diff --git a/.github/workflows/codemode-evals.yml b/.github/workflows/codemode-evals.yml index 18c7eda..05a4ed9 100644 --- a/.github/workflows/codemode-evals.yml +++ b/.github/workflows/codemode-evals.yml @@ -26,7 +26,9 @@ jobs: # unchanged main added no signal for these offline checks. deterministic: name: Deterministic evals - runs-on: macos-latest + # Pinned, not macos-latest: the package declares a macOS 15 floor, and a + # silently-rolling runner would move that floor without a code change. + runs-on: macos-15 steps: - name: Checkout uses: actions/checkout@v4 @@ -57,7 +59,7 @@ jobs: platform-build: name: Build (${{ matrix.platform }}) - runs-on: macos-latest + runs-on: macos-15 strategy: fail-fast: false matrix: @@ -107,7 +109,7 @@ jobs: # live+compare steps with a warning, instead of pretending or failing noisily. llm-live: name: LLM eval (live regression gate) - runs-on: macos-latest + runs-on: macos-15 needs: deterministic if: github.event_name == 'workflow_dispatch' env: diff --git a/README.md b/README.md index 29bf64f..261a36c 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,12 @@ let tools = CodeModeAgentTools( - `allowedHosts` restricts fetch to an explicit list (entries match the host and its subdomains, and deliberately allowlisted private hosts such as `localhost` are honored); `blockedHosts` refuses specific hosts; `NetworkAccessPolicy.permissive` restores unrestricted behavior. - Matching is by URL host only; DNS resolution is not performed, so a public hostname that resolves to a private address is not detected. Hosts that need stricter guarantees should set `allowedHosts`. +Script traffic also carries no ambient authority: + +- `network.fetch` runs on an isolated ephemeral `URLSession` with no cookie storage, no credential storage, and no shared cache — not `URLSession.shared`, which would let a script ride whatever the host app is already logged in to and let a `Set-Cookie` poison the app's cookie jar. +- Script-supplied `Cookie`, `Authorization`, and `Proxy-*` request headers are refused. Set `allowsCredentialHeaders: true` when scripts legitimately call an authenticated API. +- A host that passes its own `session` to `NetworkBridge` takes responsibility for this; requests still disable per-request cookie handling. + ```swift let tools = CodeModeAgentTools( config: CodeModeConfiguration( @@ -152,6 +158,79 @@ let tools = CodeModeAgentTools( ) ``` +## Agent Guidance + +The tool descriptions and the generated TypeScript tell a model *what the helpers +are called*. Neither tells it *what shape its work should take* — so an agent can +end up using `executeJavaScript` as a thin RPC, one call per operation, which is +the pattern this package exists to replace. + +`CodeModeAgentGuidance` is the missing half. Pair it with `typeDeclarations()`: + +```swift +let systemPrompt = CodeModeAgentGuidance.systemPrompt(.standard) + + "\n\n" + + tools.typeDeclarations() +``` + +Three tiers, each self-contained and additive — dropping to a smaller one loses +examples, never a rule: + +| Tier | ~Tokens | Contents | +| --- | --- | --- | +| `.brief` | ~160 | The core idea: one call runs the whole task; request only the capabilities you use. | +| `.standard` | ~580 | Adds the search → write → repair workflow and a worked multi-step example. | +| `.full` | ~1000 | Adds partial-failure handling, the sequential-native-call reality, and the anti-pattern list. | + +Or let a budget choose: + +```swift +CodeModeAgentGuidance.systemPrompt(approximateTokenBudget: 400) // -> .standard +``` + +Token counts are estimated at four characters per token — a planning bound, not a +real tokenization. A budget too small for any tier still returns `.brief`, on the +grounds that some guidance is the difference between an agent that batches its +work and one that does not. + +The `execution.*` and `fs.whole-job-one-script` eval scenarios grade this: a +transcript that splits one obvious job across several executions, or that asks +for capabilities it never calls, fails. + +## TypeScript Declarations + +The registry already knows argument names, types, optionality, enum constraints, +and hints. That metadata is emitted as TypeScript so the model writes code +against real declarations instead of a coarse type map plus prose: + +```swift +let tools = CodeModeAgentTools() +print(tools.typeDeclarations()) // whole platform-filtered surface +print(tools.capabilities()) // every reference, each carrying `dts` +``` + +Every `JavaScriptAPIReference` carries a per-capability `dts`, so code-driven +search stays the filter and TypeScript becomes the payload: + +```javascript +async () => { + return api.references + .filter(ref => ref.tags.includes("calendar")) + .map(ref => ref.dts) + .join("\n\n"); +} +``` + +Constrained arguments become string-literal unions, dotted argument paths +(`options.timeoutMs`) become nested object types, and argument hints become doc +comments. Results are typed `CodeModeValue` — built-in bridges return untyped +JSON dictionaries, so a narrower result type would be a fiction; the prose result +summary is in the doc comment until per-capability result schemas exist. + +The Node-compatibility globals (`fetch`, `fs.promises.*`, `path`, `console`) are +declared in a hand-authored preamble, because their positional calling convention +is not something the catalog can express. + ## Search `searchJavaScriptAPI` accepts `JavaScriptAPISearchRequest`: @@ -215,11 +294,44 @@ async () => { - `code` - `allowedCapabilities` +- `allowedCapabilityKeys` - `timeoutMs` - `context` It returns a `JavaScriptExecutionCall` immediately. +### Capability allowlists are model-authored; the grant is not + +`allowedCapabilities` and `allowedCapabilityKeys` travel in the advertised tool +schema, so the *model* fills them in. They are a least-privilege declaration and +a useful audit signal, but on their own they are not a sandbox: a host that pipes +tool JSON straight into `executeJavaScript` gives the script whatever the script +asked for. + +`CodeModeConfiguration.capabilityGrant` is the host-owned ceiling. The effective +set is always `requested ∩ granted`: + +```swift +let tools = CodeModeAgentTools( + config: CodeModeConfiguration( + capabilityGrant: .only( + [.fsRead, .fsWrite, .networkFetch], + capabilityKeys: ["myapp.tasks.complete"] + ) + ) +) +``` + +It defaults to `.unrestricted` for source compatibility. Anything the request +declares and the grant withholds fails with `CAPABILITY_DENIED`, is reported to +the model as not repairable by retrying, and is recorded as a +`CAPABILITY_WITHHELD_BY_HOST` diagnostic. + +The two allowlists are strictly disjoint. Built-in `CapabilityID`s are granted +only by `allowedCapabilities`; `allowedCapabilityKeys` accepts arbitrary strings +and reaches custom providers only, so it cannot be used to spell a built-in past +a host that vets the typed field. + Cross-platform privileged helpers are installed under `apple.*`. Platform-specific helpers are installed only where supported, for example `ios.alarm.*` on iOS hosts that support AlarmKit. `apple.location.requestPermission()` is currently exposed only on iOS hosts. Other Apple platforms can expose `apple.location.*` helpers when supported, but the explicit permission-request helper is intentionally hidden outside iOS for now. @@ -242,6 +354,25 @@ System UI helpers are installed only on supported UI platforms. Shared iOS/visio - on failure it throws `CodeModeToolError` - `call.cancel()` interrupts in-flight JavaScript +### Timers and the event loop + +`setTimeout`/`clearTimeout` are real: callbacks are deferred, delays are honoured, +`clearTimeout` cancels, and an error thrown inside a callback becomes a +`TIMER_CALLBACK_ERROR` diagnostic rather than propagating to whoever called +`setTimeout`. So `await new Promise(r => setTimeout(r, 1000))` — the standard +backoff — actually waits instead of spinning. A long wait remains cancellable and +is still bounded by `timeoutMs`. + +There is no I/O event loop yet: the bridge ABI is synchronous, so native calls run +one at a time and `Promise.all` over several helpers completes them sequentially. +Because a timer is the only thing that can advance a pending program, a promise +with no resolve path and no queued timer is provably unsettleable and fails +immediately with `JS_RUNTIME_ERROR` instead of running out the clock. + +Bridge calls that fail while the script still returns successfully — typically a +forgotten `await`, which silently discards the rejection — produce a +`BRIDGE_FAILURES_NOT_SURFACED` warning diagnostic. + ### Timeout and cancellation `timeoutMs` and `cancel()` are enforced preemptively: a JavaScriptCore execution diff --git a/Sources/CodeMode/API/BridgeModels.swift b/Sources/CodeMode/API/BridgeModels.swift index e922dd9..d86b492 100644 --- a/Sources/CodeMode/API/BridgeModels.swift +++ b/Sources/CodeMode/API/BridgeModels.swift @@ -3,7 +3,14 @@ import Foundation public struct CodeModeConfiguration: Sendable { public var pathPolicy: any PathPolicy public var networkAccessPolicy: NetworkAccessPolicy + /// Host-owned ceiling on the model-authored `allowedCapabilities` / + /// `allowedCapabilityKeys`. Defaults to `.unrestricted`; see `CapabilityGrant`. + public var capabilityGrant: CapabilityGrant public var fileSystem: any CodeModeFileSystem + /// Byte ceilings applied by the filesystem bridge to `fs.read` / `fs.write`. + public var fileSystemLimits: FileSystemLimits + /// Bounds on retained result/log/event volume for a single execution. + public var executionLimits: ExecutionLimits public var artifactStore: any ArtifactStore public var permissionBroker: any PermissionBroker public var auditLogger: any AuditLogger @@ -25,7 +32,10 @@ public struct CodeModeConfiguration: Sendable { public init( pathPolicy: any PathPolicy = DefaultPathPolicy(), networkAccessPolicy: NetworkAccessPolicy = .standard, + capabilityGrant: CapabilityGrant = .unrestricted, fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), + fileSystemLimits: FileSystemLimits = .standard, + executionLimits: ExecutionLimits = .standard, artifactStore: any ArtifactStore = InMemoryArtifactStore(), permissionBroker: any PermissionBroker = SystemPermissionBroker(), auditLogger: any AuditLogger = SyncAuditLogger(), @@ -46,7 +56,10 @@ public struct CodeModeConfiguration: Sendable { ) { self.pathPolicy = pathPolicy self.networkAccessPolicy = networkAccessPolicy + self.capabilityGrant = capabilityGrant self.fileSystem = fileSystem + self.fileSystemLimits = fileSystemLimits + self.executionLimits = executionLimits self.artifactStore = artifactStore self.permissionBroker = permissionBroker self.auditLogger = auditLogger @@ -89,6 +102,10 @@ public struct JavaScriptAPIReference: Sendable, Codable, Equatable { public var argumentHints: [String: String] public var argumentConstraints: CapabilityArgumentConstraints public var resultSummary: String + /// TypeScript declaration for this helper, generated from the fields above. + /// Return it from `searchJavaScriptAPI` to give the model real types rather + /// than a coarse `argumentTypes` map plus prose. + public var dts: String public init( capability: String, @@ -103,7 +120,8 @@ public struct JavaScriptAPIReference: Sendable, Codable, Equatable { argumentTypes: [String: CapabilityArgumentType], argumentHints: [String: String], argumentConstraints: CapabilityArgumentConstraints = .none, - resultSummary: String + resultSummary: String, + dts: String = "" ) { self.capability = capability self.capabilityKey = capabilityKey ?? CodeModeCapabilityKey(rawValue: capability) @@ -118,6 +136,7 @@ public struct JavaScriptAPIReference: Sendable, Codable, Equatable { self.argumentHints = argumentHints self.argumentConstraints = argumentConstraints self.resultSummary = resultSummary + self.dts = dts } } @@ -132,6 +151,13 @@ public struct JavaScriptAPISearchResponse: Sendable, Codable, Equatable { } public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { + /// Bounds advertised in `executeJavaScriptParameterSchema`. Values outside + /// them are clamped rather than rejected: a model that asks for a 10-minute + /// budget should get the ceiling and a running script, not a hard failure. + public static let minimumTimeoutMs = 1 + public static let maximumTimeoutMs = 60_000 + public static let defaultTimeoutMs = 10_000 + public var code: String public var allowedCapabilities: [CapabilityID] public var allowedCapabilityKeys: [CodeModeCapabilityKey] @@ -142,16 +168,20 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { code: String, allowedCapabilities: [CapabilityID], allowedCapabilityKeys: [CodeModeCapabilityKey] = [], - timeoutMs: Int = 10_000, + timeoutMs: Int = JavaScriptExecutionRequest.defaultTimeoutMs, context: ExecutionContext = .init() ) { self.code = code self.allowedCapabilities = allowedCapabilities self.allowedCapabilityKeys = allowedCapabilityKeys - self.timeoutMs = timeoutMs + self.timeoutMs = Self.clampTimeoutMs(timeoutMs) self.context = context } + static func clampTimeoutMs(_ value: Int) -> Int { + min(max(value, minimumTimeoutMs), maximumTimeoutMs) + } + private enum CodingKeys: String, CodingKey { case code case allowedCapabilities @@ -194,10 +224,13 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { self.allowedCapabilities = capabilities self.allowedCapabilityKeys = try container.decodeIfPresent([CodeModeCapabilityKey].self, forKey: .allowedCapabilityKeys) ?? [] - self.timeoutMs = try Self.decodeTimeoutMs(from: container) ?? 10_000 + self.timeoutMs = Self.clampTimeoutMs(try Self.decodeTimeoutMs(from: container) ?? Self.defaultTimeoutMs) self.context = try container.decodeIfPresent(ExecutionContext.self, forKey: .context) ?? .init() } + // Every conversion here is total. `Int.init(_: Double)` traps on non-finite + // and out-of-range input, and this runs on model-authored JSON before any + // script does — `{"timeoutMs": 1e300}` would kill the host process in-process. private static func decodeTimeoutMs(from container: KeyedDecodingContainer) throws -> Int? { guard container.contains(.timeoutMs), try !container.decodeNil(forKey: .timeoutMs) else { return nil @@ -206,11 +239,11 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { return value } if let value = try? container.decode(Double.self, forKey: .timeoutMs) { - return Int(value) + return try requireRepresentable(value, in: container) } if let string = try? container.decode(String.self, forKey: .timeoutMs), let value = Double(string.trimmingCharacters(in: .whitespaces)) { - return Int(value) + return try requireRepresentable(value, in: container) } throw DecodingError.dataCorruptedError( forKey: .timeoutMs, @@ -218,6 +251,20 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { debugDescription: "timeoutMs must be an integer or a numeric string." ) } + + private static func requireRepresentable( + _ value: Double, + in container: KeyedDecodingContainer + ) throws -> Int { + guard let converted = JSONValue.exactInt(from: value) else { + throw DecodingError.dataCorruptedError( + forKey: .timeoutMs, + in: container, + debugDescription: "timeoutMs must be a finite number representable as an integer; received \(value)." + ) + } + return converted + } } public enum JavaScriptExecutionEvent: Sendable, Equatable { @@ -314,7 +361,10 @@ public final class JavaScriptExecutionCall: @unchecked Sendable { public var result: JavaScriptExecutionResult { get async throws { let outcome = await withTaskCancellationHandler { - await waitForResultOutcome() + // `resultTask.result` already detaches the await from the calling + // task's cancellation, so the previous `Task.detached` per access + // bought nothing but an extra task. + await resultTask.result } onCancel: { self.cancel() } @@ -324,8 +374,7 @@ public final class JavaScriptExecutionCall: @unchecked Sendable { return result case let .failure(error as CodeModeToolError): throw error - case let .failure(error as CancellationError): - _ = error + case .failure(is CancellationError): throw CodeModeToolError(code: "CANCELLED", message: "Execution cancelled") case let .failure(error): throw error @@ -338,16 +387,12 @@ public final class JavaScriptExecutionCall: @unchecked Sendable { resultTask.cancel() } - private func waitForResultOutcome() async -> Result { - await withCheckedContinuation { continuation in - Task.detached { - do { - continuation.resume(returning: .success(try await self.resultTask.value)) - } catch { - continuation.resume(returning: .failure(error)) - } - } - } + /// A dropped call must not keep running: without this, an execution whose + /// handle nobody holds continues to completion in the background, still + /// touching the filesystem, the network, and system UI on the user's behalf. + deinit { + cancelImpl() + resultTask.cancel() } } diff --git a/Sources/CodeMode/API/SystemAppleServiceClients.swift b/Sources/CodeMode/API/SystemAppleServiceClients.swift index 1f3db40..c501be0 100644 --- a/Sources/CodeMode/API/SystemAppleServiceClients.swift +++ b/Sources/CodeMode/API/SystemAppleServiceClients.swift @@ -819,8 +819,14 @@ enum SystemMapsMapping { private func openSystemURL(_ url: URL) -> Bool { #if canImport(UIKit) if Thread.isMainThread { - UIApplication.shared.open(url, options: [:], completionHandler: nil) - return true + // `UIApplication.shared` and `open` are main-actor-isolated. We have + // already established we are on the main thread, so state that to the + // compiler rather than hopping — a hop here would return before the open + // has been requested. + return MainActor.assumeIsolated { + UIApplication.shared.open(url, options: [:], completionHandler: nil) + return true + } } let semaphore = DispatchSemaphore(value: 0) let opened = LockedBox(false) diff --git a/Sources/CodeMode/Bridges/AlarmBridge.swift b/Sources/CodeMode/Bridges/AlarmBridge.swift index 4736f96..6931d19 100644 --- a/Sources/CodeMode/Bridges/AlarmBridge.swift +++ b/Sources/CodeMode/Bridges/AlarmBridge.swift @@ -139,13 +139,7 @@ public final class AlarmBridge: @unchecked Sendable { } private func isoDate(_ text: String?) -> Date? { - guard let text else { return nil } - let formatter = ISO8601DateFormatter() - if let date = formatter.date(from: text) { - return date - } - formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - return formatter.date(from: text) + CodeModeDate.parse(text) } #if canImport(AlarmKit) && canImport(SwiftUI) && os(iOS) diff --git a/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift b/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift index 55090e8..1160d8c 100644 --- a/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift +++ b/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift @@ -111,7 +111,12 @@ public final class SpeechBridge: @unchecked Sendable { try ensurePermission(.speechRecognition, context: context) let path = try requireNonEmptyString("path", in: arguments, capability: "speech.file.transcribe") let resolved = try context.pathPolicy.resolve(path: path) + // The raw, script-supplied `path` is replaced rather than accompanied: + // leaving both in the dictionary made every host adapter one + // `arguments.string("path")` away from a path-policy bypass, with nothing + // in the code saying which key was the safe one. var clientArguments = arguments + clientArguments["path"] = .string(resolved.path) clientArguments["resolvedPath"] = .string(resolved.path) try validateOptionalPositiveNumber("timeoutMs", in: arguments, capability: "speech.file.transcribe") return try client.transcribeFile(arguments: clientArguments) @@ -348,7 +353,9 @@ public final class PassKitBridge: @unchecked Sendable { public func addPass(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let path = try requireNonEmptyString("path", in: arguments, capability: "passkit.pass.add") let resolved = try context.pathPolicy.resolve(path: path) + // As above: the raw path does not survive into the client dictionary. var clientArguments = arguments + clientArguments["path"] = .string(resolved.path) clientArguments["resolvedPath"] = .string(resolved.path) return try client.addPass(arguments: clientArguments) } diff --git a/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift b/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift index 706b4af..1c210e3 100644 --- a/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift +++ b/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift @@ -14,7 +14,7 @@ extension DefaultCapabilityRegistrationBuilder { descriptor: .init( id: .networkFetch, title: "Fetch HTTP resource", - summary: "Perform HTTP(S) requests through URLSession via a fetch-compatible API.", + summary: "Perform HTTP(S) requests through an isolated URLSession via a fetch-compatible API. Requests carry no app cookies or stored credentials, and credential headers (Cookie, Authorization, Proxy-*) are refused unless the host app permits them.", tags: ["network", "http", "fetch"], example: "await fetch('https://api.example.com/data').then(r => r.json())", requiredArguments: ["url"], @@ -29,7 +29,7 @@ extension DefaultCapabilityRegistrationBuilder { argumentHints: [ "url": "Absolute HTTP(S) URL string.", "options.method": "HTTP method; defaults to GET.", - "options.headers": "Object of header key/value string pairs.", + "options.headers": "Object of header key/value string pairs. Cookie, Authorization, and Proxy-* are refused unless the host app permits credential headers.", "options.body": "UTF-8 request body string.", "options.bodyBase64": "Base64-encoded request body. Mutually exclusive with options.body.", "options.timeoutMs": "Request and bridge wait timeout in milliseconds; defaults to 30000.", diff --git a/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift b/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift index 96eb81e..3985f55 100644 --- a/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift +++ b/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift @@ -3,6 +3,7 @@ import Foundation public enum DefaultCapabilityLoader { public static func loadAllRegistrations( fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), + fileSystemLimits: FileSystemLimits = .standard, networkAccessPolicy: NetworkAccessPolicy = .standard, eventInbox: any CodeModeEventInbox = UnavailableCodeModeEventInbox(), cloudKitClient: any CloudKitClient = UnavailableCloudKitClient(), @@ -18,6 +19,7 @@ public enum DefaultCapabilityLoader { ) -> [CapabilityRegistration] { DefaultCapabilityRegistrationBuilder( fileSystem: fileSystem, + fileSystemLimits: fileSystemLimits, networkAccessPolicy: networkAccessPolicy, eventInbox: eventInbox, cloudKitClient: cloudKitClient, @@ -64,6 +66,7 @@ struct DefaultCapabilityRegistrationBuilder { init( fileSystem: any CodeModeFileSystem, + fileSystemLimits: FileSystemLimits, networkAccessPolicy: NetworkAccessPolicy, eventInbox: any CodeModeEventInbox, cloudKitClient: any CloudKitClient, @@ -77,7 +80,7 @@ struct DefaultCapabilityRegistrationBuilder { passKitClient: any PassKitClient, storeKitClient: any StoreKitClient ) { - self.fs = FileSystemBridge(fileSystem: fileSystem) + self.fs = FileSystemBridge(fileSystem: fileSystem, limits: fileSystemLimits) self.network = NetworkBridge(policy: networkAccessPolicy) self.keychain = KeychainBridge() self.location = LocationBridge() @@ -91,7 +94,7 @@ struct DefaultCapabilityRegistrationBuilder { self.health = HealthBridge() self.home = HomeBridge() self.media = MediaBridge() - self.systemUI = SystemUIBridge() + self.systemUI = SystemUIBridge(networkAccessPolicy: networkAccessPolicy) self.eventInbox = eventInbox self.cloudKit = CloudKitBridge(client: cloudKitClient, eventInbox: eventInbox) self.remoteNotifications = RemoteNotificationsBridge(client: remoteNotificationsClient, eventInbox: eventInbox) diff --git a/Sources/CodeMode/Bridges/EventKitBridge.swift b/Sources/CodeMode/Bridges/EventKitBridge.swift index 279c31b..9750c4a 100644 --- a/Sources/CodeMode/Bridges/EventKitBridge.swift +++ b/Sources/CodeMode/Bridges/EventKitBridge.swift @@ -5,6 +5,8 @@ import EventKit #endif public final class EventKitBridge: @unchecked Sendable { + static let reminderFetchTimeoutSeconds: TimeInterval = 15 + public init() {} public func readEvents(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { @@ -15,8 +17,13 @@ public final class EventKitBridge: @unchecked Sendable { #if canImport(EventKit) let store = EKEventStore() - let start = isoDate(arguments.string("start")) ?? Date() - let end = isoDate(arguments.string("end")) ?? Calendar.current.date(byAdding: .day, value: 14, to: start) ?? start + // A value that is present but unparseable is an error, not a reason to + // fall back to now/+14d: that handed the script a plausible but wrong + // window with no diagnostic to notice it by. + let start = try CodeModeDate.optional(arguments.string("start"), argument: "start", capability: "calendar.read") ?? Date() + let end = try CodeModeDate.optional(arguments.string("end"), argument: "end", capability: "calendar.read") + ?? Calendar.current.date(byAdding: .day, value: 14, to: start) + ?? start let limit = arguments.int("limit") ?? 50 let calendars = try resolveCalendars( from: arguments, @@ -97,11 +104,9 @@ public final class EventKitBridge: @unchecked Sendable { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - var result: [JSONValue] = [] let includeCompleted = arguments.bool("includeCompleted") ?? false - let start = isoDate(arguments.string("start")) - let end = isoDate(arguments.string("end")) + let start = try CodeModeDate.optional(arguments.string("start"), argument: "start", capability: "reminders.read") + let end = try CodeModeDate.optional(arguments.string("end"), argument: "end", capability: "reminders.read") let limit = max(1, arguments.int("limit") ?? 50) let calendars = try resolveCalendars( from: arguments, @@ -113,23 +118,30 @@ public final class EventKitBridge: @unchecked Sendable { let predicate = includeCompleted ? store.predicateForReminders(in: calendars) : store.predicateForIncompleteReminders(withDueDateStarting: start, ending: end, calendars: calendars) - store.fetchReminders(matching: predicate) { reminders in - let filtered = (reminders ?? []) - .filter { reminder in - guard includeCompleted else { return true } - guard let dueDate = reminder.dueDateComponents?.date else { - return start == nil && end == nil + // Previously a `var` written from EventKit's completion queue, read after + // a semaphore wait whose timeout result was discarded — a data race with + // the late callback, and an empty array on timeout that the script could + // not tell apart from "no reminders". + let result = try CompletionWait.value( + timeout: Self.reminderFetchTimeoutSeconds, + operationName: "reminders.read" + ) { complete in + store.fetchReminders(matching: predicate) { reminders in + let filtered = (reminders ?? []) + .filter { reminder in + guard includeCompleted else { return true } + guard let dueDate = reminder.dueDateComponents?.date else { + return start == nil && end == nil + } + if let start, dueDate < start { return false } + if let end, dueDate > end { return false } + return true } - if let start, dueDate < start { return false } - if let end, dueDate > end { return false } - return true - } - .prefix(limit) - result = filtered.map { Self.reminderJSON($0) } - semaphore.signal() + .prefix(limit) + complete(filtered.map { Self.reminderJSON($0) }) + } } - _ = semaphore.wait(timeout: .now() + 15) return .array(result) #else _ = arguments @@ -191,8 +203,7 @@ public final class EventKitBridge: @unchecked Sendable { } private func isoDate(_ text: String?) -> Date? { - guard let text, text.isEmpty == false else { return nil } - return ISO8601DateFormatter().date(from: text) + CodeModeDate.parse(text) } private func eventOperation(_ arguments: [String: JSONValue]) throws -> CalendarWriteOperation { @@ -480,6 +491,17 @@ public final class EventKitBridge: @unchecked Sendable { return span.ekSpan } + /// Test seam: these serializers are the only EventKit logic reachable + /// without a live, permission-granted store, and they are where the + /// implicitly-unwrapped `calendar`/`title` crashes lived. + static func eventJSONForTesting(_ event: EKEvent) -> JSONValue { + eventJSON(event) + } + + static func reminderJSONForTesting(_ reminder: EKReminder) -> JSONValue { + reminderJSON(reminder) + } + private static func eventJSON(_ event: EKEvent) -> JSONValue { .object([ "identifier": .string(event.eventIdentifier ?? ""), @@ -487,8 +509,11 @@ public final class EventKitBridge: @unchecked Sendable { "startDate": .string(event.startDate.ISO8601Format()), "endDate": .string(event.endDate.ISO8601Format()), "notes": .string(event.notes ?? ""), - "calendarIdentifier": .string(event.calendar.calendarIdentifier), - "calendarTitle": .string(event.calendar.title), + // `EKEvent.calendar` is `EKCalendar!` and is nil for an orphaned + // event — every neighboring field here is nil-coalesced, and this one + // crashed the execution thread. + "calendarIdentifier": .string(event.calendar?.calendarIdentifier ?? ""), + "calendarTitle": .string(event.calendar?.title ?? ""), "location": .string(event.location ?? ""), "url": .string(event.url?.absoluteString ?? ""), "isAllDay": .bool(event.isAllDay), @@ -498,7 +523,8 @@ public final class EventKitBridge: @unchecked Sendable { private static func reminderJSON(_ reminder: EKReminder) -> JSONValue { var object: [String: JSONValue] = [ "identifier": .string(reminder.calendarItemIdentifier), - "title": .string(reminder.title), + // `EKReminder.title` is `String!`. + "title": .string(reminder.title ?? ""), "isCompleted": .bool(reminder.isCompleted), "dueDate": .string(reminder.dueDateComponents?.date?.ISO8601Format() ?? ""), "completionDate": .string(reminder.completionDate?.ISO8601Format() ?? ""), diff --git a/Sources/CodeMode/Bridges/FileSystemBridge.swift b/Sources/CodeMode/Bridges/FileSystemBridge.swift index a4b8be9..1118038 100644 --- a/Sources/CodeMode/Bridges/FileSystemBridge.swift +++ b/Sources/CodeMode/Bridges/FileSystemBridge.swift @@ -1,14 +1,40 @@ import Foundation +/// Byte ceilings for the filesystem bridge. +/// +/// A sandbox root can hold files far larger than a script's result budget — a +/// 2 GB video in `documents:` read whole and base64-serialized is an immediate +/// jetsam on iOS — so reads and writes are bounded and fail with a diagnostic +/// instead of exhausting memory. +public struct FileSystemLimits: Sendable, Equatable { + public var maxReadBytes: Int + public var maxWriteBytes: Int + + public init(maxReadBytes: Int = 16 * 1_024 * 1_024, maxWriteBytes: Int = 16 * 1_024 * 1_024) { + self.maxReadBytes = maxReadBytes + self.maxWriteBytes = maxWriteBytes + } + + public static let standard = FileSystemLimits() + + /// No ceiling. Only appropriate when the host has its own quota. + public static let unlimited = FileSystemLimits(maxReadBytes: .max, maxWriteBytes: .max) +} + public final class FileSystemBridge: @unchecked Sendable { private let fileSystem: any CodeModeFileSystem + private let limits: FileSystemLimits - public init(fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem()) { + public init( + fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), + limits: FileSystemLimits = .standard + ) { self.fileSystem = fileSystem + self.limits = limits } - public convenience init(fileManager: FileManager) { - self.init(fileSystem: LocalCodeModeFileSystem(fileManager: fileManager)) + public convenience init(fileManager: FileManager, limits: FileSystemLimits = .standard) { + self.init(fileSystem: LocalCodeModeFileSystem(fileManager: fileManager), limits: limits) } public func list(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { @@ -36,11 +62,23 @@ public final class FileSystemBridge: @unchecked Sendable { let encoding = arguments.string("encoding") ?? "utf8" let url = try context.pathPolicy.resolve(path: path) + + // Checked against stat first: reading and then rejecting has already spent + // the memory the cap exists to protect. Re-checked after, because a + // custom CodeModeFileSystem's stat may be absent or stale. + try enforceReadLimit(try? fileSystem.attributesOfItem(at: url).size, at: url) let data = try fileSystem.readData(at: url) + try enforceReadLimit(data.count, at: url) switch encoding.lowercased() { case "utf8", "utf-8": - let text = String(data: data, encoding: .utf8) ?? "" + // Silently substituting "" for undecodable bytes tells the script the + // file is empty, which is worse than a failure it can act on. + guard let text = String(data: data, encoding: .utf8) else { + throw BridgeError.invalidArguments( + "fs.read could not decode \(url.lastPathComponent) as UTF-8. Retry with encoding: 'base64' for binary data." + ) + } return .object([ "path": .string(url.path), "text": .string(text), @@ -63,9 +101,6 @@ public final class FileSystemBridge: @unchecked Sendable { let encoding = arguments.string("encoding") ?? "utf8" let url = try context.pathPolicy.resolve(path: path) - let parent = url.deletingLastPathComponent() - try fileSystem.createDirectory(at: parent, recursive: true) - let data: Data switch encoding.lowercased() { case "utf8", "utf-8": @@ -80,6 +115,13 @@ public final class FileSystemBridge: @unchecked Sendable { throw BridgeError.invalidArguments("Unsupported encoding: \(encoding)") } + guard data.count <= limits.maxWriteBytes else { + throw Self.overLimit("fs.write", url: url, bytes: data.count, limit: limits.maxWriteBytes) + } + + // Only after the payload is known good, so a rejected write leaves no + // directories behind. + try fileSystem.createDirectory(at: url.deletingLastPathComponent(), recursive: true) try fileSystem.writeData(data, to: url) return .object([ "path": .string(url.path), @@ -94,10 +136,7 @@ public final class FileSystemBridge: @unchecked Sendable { let fromURL = try context.pathPolicy.resolve(path: from) let toURL = try context.pathPolicy.resolve(path: to) - - if fileSystem.itemExists(at: toURL) { - try fileSystem.removeItem(at: toURL) - } + try prepareDestination(toURL, operation: "fs.move", arguments: arguments, context: context) try fileSystem.moveItem(at: fromURL, to: toURL) return .object([ @@ -113,10 +152,7 @@ public final class FileSystemBridge: @unchecked Sendable { let fromURL = try context.pathPolicy.resolve(path: from) let toURL = try context.pathPolicy.resolve(path: to) - - if fileSystem.itemExists(at: toURL) { - try fileSystem.removeItem(at: toURL) - } + try prepareDestination(toURL, operation: "fs.copy", arguments: arguments, context: context) try fileSystem.copyItem(at: fromURL, to: toURL) return .object([ @@ -125,6 +161,59 @@ public final class FileSystemBridge: @unchecked Sendable { ]) } + private func enforceReadLimit(_ bytes: Int?, at url: URL) throws { + guard let bytes, bytes > limits.maxReadBytes else { + return + } + throw Self.overLimit("fs.read", url: url, bytes: bytes, limit: limits.maxReadBytes) + } + + private static func overLimit(_ operation: String, url: URL, bytes: Int, limit: Int) -> BridgeError { + .invalidArguments( + "\(operation) refused \(url.lastPathComponent): \(bytes) bytes exceeds the \(limit)-byte limit. Use a smaller payload, or have the host raise CodeModeConfiguration.fileSystemLimits." + ) + } + + /// Clears the way for a move/copy, refusing anything destructive the caller + /// did not explicitly ask for. + /// + /// The previous behavior was an unconditional recursive `removeItem` on any + /// existing destination, and containment alone admits a root: `documents:` + /// resolves to the documents root, so `fs.move({ from: 'tmp:junk.txt', + /// to: 'documents:' })` deleted the user's whole Documents directory. The + /// gates here match the ones `fs.delete` already enforces. + private func prepareDestination( + _ toURL: URL, + operation: String, + arguments: [String: JSONValue], + context: BridgeInvocationContext + ) throws { + guard context.pathPolicy.isAllowedRoot(toURL) == false else { + throw BridgeError.pathViolation( + "\(operation) refuses a sandbox root as its destination. Name a path inside the root, for example 'documents:archive/file.txt'." + ) + } + + guard fileSystem.itemExists(at: toURL) else { + return + } + + guard arguments.bool("overwrite") == true else { + throw BridgeError.invalidArguments( + "\(operation) destination already exists. Pass overwrite: true to replace it, or choose a different 'to' path." + ) + } + + let isDirectory = (try? fileSystem.attributesOfItem(at: toURL))?.isDirectory ?? false + if isDirectory, arguments.bool("recursive") != true { + throw BridgeError.invalidArguments( + "\(operation) destination is a directory. Pass recursive: true along with overwrite: true to replace it and everything under it." + ) + } + + try fileSystem.removeItem(at: toURL) + } + public func delete(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { guard let path = arguments.string("path") else { throw BridgeError.invalidArguments("fs.delete requires 'path'") @@ -137,6 +226,12 @@ public final class FileSystemBridge: @unchecked Sendable { return .object(["deleted": .bool(false), "path": .string(url.path)]) } + guard context.pathPolicy.isAllowedRoot(url) == false else { + throw BridgeError.pathViolation( + "fs.delete refuses a sandbox root. Delete entries inside it individually." + ) + } + let attrs = try fileSystem.attributesOfItem(at: url) if attrs.isDirectory, recursive == false { throw BridgeError.invalidArguments("fs.delete requires recursive=true for directories") diff --git a/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift b/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift index 650190b..ca296cb 100644 --- a/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift +++ b/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift @@ -1,7 +1,16 @@ import Foundation -// Filesystem capabilities. Flat arguments, no constrained values, so no -// CodeModeStringEnum is needed here. +// Filesystem capabilities. + +/// The encodings `fs.read`/`fs.write` accept. Declared as an enum so the catalog +/// advertises `allowedStringValues` instead of leaving the constraint in prose, +/// where the model cannot discover it. +enum FileEncoding: String, CodeModeStringEnum { + case utf8 + case base64 + + static let codeModeAliases: [String: FileEncoding] = ["utf-8": .utf8] +} @BuiltInCodeMode(.fsList, path: "apple.fs.list", aliases: ["fs.promises.readdir"]) struct FSListTool: BuiltInCodeModeTool { @@ -35,8 +44,8 @@ struct FSReadTool: BuiltInCodeModeTool { struct Arguments: Sendable { @ToolParam("File path using allowed root prefix.") var path: String - @ToolParam("utf8 (default) or base64.") - var encoding: String? + @ToolParam("utf8 (default) or base64. Binary files must use base64; a utf8 read of undecodable bytes fails rather than returning empty text.") + var encoding: FileEncoding? var raw: [String: JSONValue] } @@ -61,7 +70,7 @@ struct FSWriteTool: BuiltInCodeModeTool { @ToolParam("UTF-8 text or base64 string depending on encoding.") var data: String? @ToolParam("utf8 (default) or base64.") - var encoding: String? + var encoding: FileEncoding? var raw: [String: JSONValue] } @@ -75,7 +84,7 @@ struct FSWriteTool: BuiltInCodeModeTool { @BuiltInCodeMode(.fsMove, path: "apple.fs.move", aliases: ["fs.promises.rename"]) struct FSMoveTool: BuiltInCodeModeTool { static let codeModeTitle = "Move file" - static let codeModeSummary = "Move file/directory within allowed sandbox roots." + static let codeModeSummary = "Move file/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination." static let codeModeTags = ["filesystem", "io", "fs"] static let codeModeExample = "await apple.fs.move({ from: 'tmp:a.txt', to: 'tmp:b.txt' })" static let codeModeResultSummary = "Object with from/to resolved paths." @@ -83,8 +92,12 @@ struct FSMoveTool: BuiltInCodeModeTool { struct Arguments: Sendable { @ToolParam("Source sandbox path.") var from: String - @ToolParam("Destination sandbox path.") + @ToolParam("Destination sandbox path. Must name a path inside a root, not the root itself.") var to: String + @ToolParam("Required as true to replace an existing destination; default false.") + var overwrite: Bool? + @ToolParam("Required as true alongside overwrite when the existing destination is a directory.") + var recursive: Bool? var raw: [String: JSONValue] } @@ -98,7 +111,7 @@ struct FSMoveTool: BuiltInCodeModeTool { @BuiltInCodeMode(.fsCopy, path: "apple.fs.copy", aliases: ["fs.promises.copyFile"]) struct FSCopyTool: BuiltInCodeModeTool { static let codeModeTitle = "Copy file" - static let codeModeSummary = "Copy file/directory within allowed sandbox roots." + static let codeModeSummary = "Copy file/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination." static let codeModeTags = ["filesystem", "io", "fs"] static let codeModeExample = "await apple.fs.copy({ from: 'tmp:a.txt', to: 'tmp:b.txt' })" static let codeModeResultSummary = "Object with from/to resolved paths." @@ -106,8 +119,12 @@ struct FSCopyTool: BuiltInCodeModeTool { struct Arguments: Sendable { @ToolParam("Source sandbox path.") var from: String - @ToolParam("Destination sandbox path.") + @ToolParam("Destination sandbox path. Must name a path inside a root, not the root itself.") var to: String + @ToolParam("Required as true to replace an existing destination; default false.") + var overwrite: Bool? + @ToolParam("Required as true alongside overwrite when the existing destination is a directory.") + var recursive: Bool? var raw: [String: JSONValue] } diff --git a/Sources/CodeMode/Bridges/HealthBridge.swift b/Sources/CodeMode/Bridges/HealthBridge.swift index 0732e72..7f584ba 100644 --- a/Sources/CodeMode/Bridges/HealthBridge.swift +++ b/Sources/CodeMode/Bridges/HealthBridge.swift @@ -26,11 +26,21 @@ public final class HealthBridge: @unchecked Sendable { let writeTypes = try requestedWriteTypes(from: writeNames) try ensureAuthorization(toShare: writeTypes, read: readTypes, context: context, forceRequest: true) + // `requestAuthorization`'s `success` means the sheet flow completed, not + // that anything was granted — HealthKit deliberately refuses to disclose + // read authorization so an app cannot infer a health condition from a + // denial. Reporting `granted: true` here was a fabrication that left the + // transcript claiming access while reads returned []. Share types are the + // only ones with an answerable status, so those are the only ones + // reported. return .object([ - "status": .string(PermissionStatus.granted.rawValue), - "granted": .bool(true), + "status": .string("completed"), + "authorizationFlowCompleted": .bool(true), "readTypes": .array(readNames.map(JSONValue.string)), "writeTypes": .array(writeNames.map(JSONValue.string)), + "writeAuthorization": .object(writeAuthorizationStatuses(for: writeNames)), + "readAuthorizationDisclosed": .bool(false), + "note": .string("HealthKit does not disclose read authorization. An empty health.read result may mean the user denied that type rather than that no samples exist."), ]) #else _ = arguments @@ -150,15 +160,7 @@ public final class HealthBridge: @unchecked Sendable { } private func parseISODate(_ value: String?) -> Date? { - guard let value else { return nil } - - let formatter = ISO8601DateFormatter() - if let date = formatter.date(from: value) { - return date - } - - formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - return formatter.date(from: value) + CodeModeDate.parse(value) } #if canImport(HealthKit) @@ -320,6 +322,44 @@ public final class HealthBridge: @unchecked Sendable { return types } + /// Per-type share authorization, which is the only part HealthKit answers + /// honestly. Read authorization is deliberately undisclosed by the framework. + private func writeAuthorizationStatuses(for names: [String]) -> [String: JSONValue] { + var statuses: [String: JSONValue] = [:] + for name in names { + guard let spec = try? writeSpec(for: name), + let quantityType = HKObjectType.quantityType(forIdentifier: spec.identifier) + else { + continue + } + statuses[name] = .string(Self.shareStatusName(healthStore.authorizationStatus(for: quantityType))) + } + return statuses + } + + /// `.granted` only when every requested share type is actually authorized; + /// otherwise `.notDetermined`, because HealthKit will not say more. + private func resolvedShareAuthorization(for toShare: Set) -> PermissionStatus { + guard toShare.isEmpty == false else { + return .notDetermined + } + let allShared = toShare.allSatisfy { healthStore.authorizationStatus(for: $0) == .sharingAuthorized } + return allShared ? .granted : .notDetermined + } + + private static func shareStatusName(_ status: HKAuthorizationStatus) -> String { + switch status { + case .sharingAuthorized: + return PermissionStatus.granted.rawValue + case .sharingDenied: + return PermissionStatus.denied.rawValue + case .notDetermined: + return PermissionStatus.notDetermined.rawValue + @unknown default: + return PermissionStatus.unavailable.rawValue + } + } + private func ensureAuthorization( toShare: Set, read: Set, @@ -340,7 +380,12 @@ public final class HealthBridge: @unchecked Sendable { let requestStatus = try authorizationRequestStatus(toShare: toShare, read: read) switch requestStatus { case .unnecessary: - context.recordPermission(.healthKit, status: .granted) + // `.unnecessary` means "there is nothing left to ask the user", not + // "access was granted" — for read types the user may well have + // denied. Record what is actually known: granted only when every + // share type says so, otherwise leave it undetermined rather than + // writing a granted claim into the transcript. + context.recordPermission(.healthKit, status: resolvedShareAuthorization(for: toShare)) case .shouldRequest, .unknown: context.recordPermission(.healthKit, status: .notDetermined) let granted = try requestAuthorization(toShare: toShare, read: read) diff --git a/Sources/CodeMode/Bridges/NetworkBridge.swift b/Sources/CodeMode/Bridges/NetworkBridge.swift index ef33703..01234a9 100644 --- a/Sources/CodeMode/Bridges/NetworkBridge.swift +++ b/Sources/CodeMode/Bridges/NetworkBridge.swift @@ -6,7 +6,27 @@ public final class NetworkBridge: @unchecked Sendable { private let session: URLSession private let policy: NetworkAccessPolicy - public init(session: URLSession = .shared, policy: NetworkAccessPolicy = .standard) { + /// The session `network.fetch` uses unless the host supplies its own. + /// + /// Deliberately *not* `URLSession.shared`: that session reads + /// `HTTPCookieStorage.shared` and `URLCredentialStorage.shared`, so every + /// script request would ride whatever the host app is already logged in to, + /// and a `Set-Cookie` in a script-fetched response would poison the app's + /// cookie jar. The sandbox's premise is that script traffic carries no + /// ambient authority, so this session has no cookie jar, no credential + /// store, and no shared cache. + public static let isolatedSession: URLSession = { + let configuration = URLSessionConfiguration.ephemeral + configuration.httpCookieStorage = nil + configuration.urlCredentialStorage = nil + configuration.urlCache = nil + configuration.httpShouldSetCookies = false + configuration.httpCookieAcceptPolicy = .never + configuration.requestCachePolicy = .reloadIgnoringLocalCacheData + return URLSession(configuration: configuration) + }() + + public init(session: URLSession = NetworkBridge.isolatedSession, policy: NetworkAccessPolicy = .standard) { self.session = session self.policy = policy } @@ -43,12 +63,23 @@ public final class NetworkBridge: @unchecked Sendable { var request = URLRequest(url: url) request.httpMethod = options.string("method")?.uppercased() ?? "GET" request.timeoutInterval = TimeInterval(timeoutMs) / 1_000 + // Belt and braces alongside the isolated session: even a host-supplied + // session must not attach its cookie jar to script traffic. + request.httpShouldHandleCookies = false if let headers = options.object("headers") { - for (key, value) in headers { - if let string = value.stringValue { - request.setValue(string, forHTTPHeaderField: key) + for key in headers.keys.sorted() { + guard let string = headers[key]?.stringValue else { + continue } + if let reason = policy.headerViolationReason(for: key) { + context.auditLogger.log(AuditEvent( + capability: CapabilityID.networkFetch.rawValue, + message: "denied \(urlString): \(reason)" + )) + throw BridgeError.networkPolicyViolation(reason) + } + request.setValue(string, forHTTPHeaderField: key) } } diff --git a/Sources/CodeMode/Bridges/NotificationsBridge.swift b/Sources/CodeMode/Bridges/NotificationsBridge.swift index da6fc2a..178f605 100644 --- a/Sources/CodeMode/Bridges/NotificationsBridge.swift +++ b/Sources/CodeMode/Bridges/NotificationsBridge.swift @@ -228,7 +228,7 @@ public final class NotificationsBridge: @unchecked Sendable { private func isoDate(_ text: String?) -> Date? { guard let text else { return nil } - return ISO8601DateFormatter().date(from: text) + return CodeModeDate.parse(text) } #if canImport(UserNotifications) diff --git a/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift b/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift index 194d853..0066ce5 100644 --- a/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift +++ b/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift @@ -39,7 +39,7 @@ private struct KeychainReadTool: BuiltInCodeModeTool { static let codeModeTitle = "Read Keychain value" static let codeModeSummary = "Read a string value from app-scoped Keychain storage." static let codeModeTags = ["security", "token", "keychain"] - static let codeModeExample = "await apple.keychain.get('auth_token')" + static let codeModeExample = "await apple.keychain.get({ key: 'auth_token' })" static let codeModeArguments = [ BuiltInToolArgument("key", .string, hint: "Logical key for this secret value."), ] @@ -67,7 +67,7 @@ private struct KeychainWriteTool: BuiltInCodeModeTool { static let codeModeTitle = "Write Keychain value" static let codeModeSummary = "Store or update a string value in app-scoped Keychain storage." static let codeModeTags = ["security", "token", "keychain"] - static let codeModeExample = "await apple.keychain.set('auth_token', token)" + static let codeModeExample = "await apple.keychain.set({ key: 'auth_token', value: token })" static let codeModeArguments = [ BuiltInToolArgument("key", .string, hint: "Logical key for this secret value."), BuiltInToolArgument("value", .string, optional: true, hint: "Secret string value. Defaults to empty string when omitted."), @@ -102,7 +102,7 @@ private struct KeychainDeleteTool: BuiltInCodeModeTool { static let codeModeTitle = "Delete Keychain value" static let codeModeSummary = "Delete an app-scoped Keychain value." static let codeModeTags = ["security", "token", "keychain"] - static let codeModeExample = "await apple.keychain.delete('auth_token')" + static let codeModeExample = "await apple.keychain.delete({ key: 'auth_token' })" static let codeModeArguments = [ BuiltInToolArgument("key", .string, hint: "Logical key for value removal."), ] diff --git a/Sources/CodeMode/Bridges/SystemUIBridge.swift b/Sources/CodeMode/Bridges/SystemUIBridge.swift index a5b59bb..1d63159 100644 --- a/Sources/CodeMode/Bridges/SystemUIBridge.swift +++ b/Sources/CodeMode/Bridges/SystemUIBridge.swift @@ -1,7 +1,11 @@ import Foundation public final class SystemUIBridge: @unchecked Sendable { - public init() {} + private let networkAccessPolicy: NetworkAccessPolicy + + public init(networkAccessPolicy: NetworkAccessPolicy = .standard) { + self.networkAccessPolicy = networkAccessPolicy + } public func pickCalendar(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { try validateCalendarPickerArguments(arguments) @@ -188,12 +192,11 @@ public final class SystemUIBridge: @unchecked Sendable { } private func validateCalendarEventArguments(_ arguments: [String: JSONValue]) throws { - let formatter = ISO8601DateFormatter() for key in ["start", "end"] { guard let value = arguments.string(key) else { continue } - guard formatter.date(from: value) != nil else { + guard CodeModeDate.parse(value) != nil else { throw BridgeError.invalidArguments("calendar.ui.presentNewEvent requires \(key) to be an ISO8601 timestamp when provided") } } @@ -447,6 +450,13 @@ public final class SystemUIBridge: @unchecked Sendable { else { throw BridgeError.invalidArguments("\(capability) \(key) must be an absolute HTTP(S) URL") } + + // Destinations opened in a browser were exempt from the policy that + // governs fetch, so a script blocked from reaching an origin could still + // put it in front of the user — or start an OAuth flow against it. + if let reason = networkAccessPolicy.violationReason(for: url) { + throw BridgeError.networkPolicyViolation("\(capability) \(key): \(reason)") + } } private func validateNonemptyString(_ arguments: [String: JSONValue], key: String, capability: String) throws { diff --git a/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift b/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift index b4d47d7..398af2d 100644 --- a/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift +++ b/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift @@ -327,7 +327,7 @@ struct WebUIPresentTool: BuiltInCodeModeTool { @BuiltInCodeMode(.authUIWebAuthenticate, path: "apple.auth.webAuthenticate") struct AuthUIWebAuthenticateTool: BuiltInCodeModeTool { static let codeModeTitle = "Authenticate with system web UI" - static let codeModeSummary = "Start an ASWebAuthenticationSession for OAuth-style browser authentication." + static let codeModeSummary = "Start an ASWebAuthenticationSession for OAuth-style browser authentication. Runs in a private browser session by default and the URL must satisfy the host's network access policy." static let codeModeTags = ["auth", "oauth", "web", "browser", "system-ui"] static let codeModeExample = "await apple.auth.webAuthenticate({ url: 'https://example.com/oauth', callbackURLScheme: 'myapp' })" static let codeModeResultSummary = "Object with action callback/cancelled and callbackURL when available." @@ -337,7 +337,7 @@ struct AuthUIWebAuthenticateTool: BuiltInCodeModeTool { var url: String @ToolParam("Optional custom URL scheme that completes the session.") var callbackURLScheme: String? - @ToolParam("Whether to prefer a private browser session; default false.") + @ToolParam("Whether to use a private browser session; default true. Set false only when the flow must reuse the user's existing browser session.") var prefersEphemeralSession: Bool? @ToolParam("Optional timeout for waiting on callback/cancellation.") var timeoutMs: Double? diff --git a/Sources/CodeMode/Catalog/BridgeCatalog.swift b/Sources/CodeMode/Catalog/BridgeCatalog.swift index 9e55914..3583424 100644 --- a/Sources/CodeMode/Catalog/BridgeCatalog.swift +++ b/Sources/CodeMode/Catalog/BridgeCatalog.swift @@ -1,25 +1,54 @@ import Foundation -struct BridgeCatalog: Sendable { +/// The model-facing view of the registry. +/// +/// Tracks the registry rather than snapshotting it once at init. The snapshot +/// used to be frozen in `CodeModeAgentTools.init`, which made +/// `CapabilityRegistry`'s public post-init `register(...)` methods either +/// unreachable in the supported flow or — if reached — a way to desync what +/// search advertises from what execution can actually invoke. +final class BridgeCatalog: @unchecked Sendable { private struct SearchCatalogPayload: Sendable, Codable { var references: [JavaScriptAPIReference] var byCapability: [String: JavaScriptAPIReference] var byJSName: [String: JavaScriptAPIReference] } - private let references: [JavaScriptAPIReference] - private let referencesByCapability: [CodeModeCapabilityKey: JavaScriptAPIReference] - private let searchCatalog: JSONValue - private let allJavaScriptNames: [String] + private struct Snapshot { + var generation: Int + var references: [JavaScriptAPIReference] + var referencesByCapability: [CodeModeCapabilityKey: JavaScriptAPIReference] + var searchCatalog: JSONValue + var allJavaScriptNames: [String] + } + + private let registry: CapabilityRegistry + private let lock = NSLock() + private var snapshot: Snapshot init(registry: CapabilityRegistry) { + self.registry = registry + self.snapshot = Self.makeSnapshot(registry: registry) + } + + /// Rebuilds if the registry has changed since the last read. Rebuilding is + /// the whole catalog, but it only happens when a provider is actually + /// registered — not on every search. + private func current() -> Snapshot { + lock.lock() + defer { lock.unlock() } + let generation = registry.generation + if snapshot.generation != generation { + snapshot = Self.makeSnapshot(registry: registry) + } + return snapshot + } + + private static func makeSnapshot(registry: CapabilityRegistry) -> Snapshot { + let generation = registry.generation let functions = registry.allRegisteredFunctions().sorted { $0.catalogCapability < $1.catalogCapability } let references = functions.map(Self.reference(from:)) - self.references = references - self.referencesByCapability = Dictionary(uniqueKeysWithValues: references.map { ($0.capabilityKey, $0) }) - self.allJavaScriptNames = Array(Set(references.flatMap(\.jsNames))).sorted() - var byJSName: [String: JavaScriptAPIReference] = [:] for reference in references { for jsName in reference.jsNames { @@ -27,29 +56,41 @@ struct BridgeCatalog: Sendable { } } - self.searchCatalog = Self.jsonValue( - from: SearchCatalogPayload( - references: references, - byCapability: Dictionary(uniqueKeysWithValues: references.map { ($0.capability, $0) }), - byJSName: byJSName - ) + return Snapshot( + generation: generation, + references: references, + referencesByCapability: Dictionary(references.map { ($0.capabilityKey, $0) }, uniquingKeysWith: { first, _ in first }), + searchCatalog: Self.jsonValue( + from: SearchCatalogPayload( + references: references, + byCapability: Dictionary(references.map { ($0.capability, $0) }, uniquingKeysWith: { first, _ in first }), + byJSName: byJSName + ) + ), + allJavaScriptNames: Array(Set(references.flatMap(\.jsNames))).sorted() ) } func reference(for capability: CapabilityID) -> JavaScriptAPIReference? { - referencesByCapability[capability.codeModeKey] + current().referencesByCapability[capability.codeModeKey] } func reference(for capabilityKey: CodeModeCapabilityKey) -> JavaScriptAPIReference? { - referencesByCapability[capabilityKey] + current().referencesByCapability[capabilityKey] } func allReferences() -> [JavaScriptAPIReference] { - references + current().references } func searchCatalogValue() -> JSONValue { - searchCatalog + current().searchCatalog + } + + /// TypeScript declarations for the whole platform-filtered surface, for a + /// host to drop into its system prompt. + func typeDeclarations() -> String { + TypeScriptDeclarations.surface(for: current().references) } func closestFunctionNames(to candidate: String, limit: Int = 3) -> [String] { @@ -62,7 +103,7 @@ struct BridgeCatalog: Sendable { let distanceThreshold = max(2, min(8, normalizedCandidate.count / 3 + 1)) - let ranked = allJavaScriptNames.map { name in + let ranked = current().allJavaScriptNames.map { name in let normalizedName = name.lowercased() let distance = Self.levenshtein(normalizedCandidate, normalizedName) let prefixBonus = normalizedName.hasPrefix(normalizedCandidate) || normalizedCandidate.hasPrefix(normalizedName) ? -2 : 0 @@ -99,7 +140,7 @@ struct BridgeCatalog: Sendable { } private static func reference(from function: RegisteredCodeModeFunction) -> JavaScriptAPIReference { - return JavaScriptAPIReference( + let reference = JavaScriptAPIReference( capability: function.catalogCapability, capabilityKey: function.capabilityKey, builtInCapability: function.builtInCapability, @@ -114,6 +155,9 @@ struct BridgeCatalog: Sendable { argumentConstraints: function.argumentConstraints, resultSummary: function.resultSummary ) + var withTypes = reference + withTypes.dts = TypeScriptDeclarations.declaration(for: reference) + return withTypes } private static func levenshtein(_ lhs: String, _ rhs: String) -> Int { diff --git a/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift b/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift new file mode 100644 index 0000000..f3a2731 --- /dev/null +++ b/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift @@ -0,0 +1,340 @@ +import Foundation + +/// Generates TypeScript declarations for the CodeMode JavaScript surface. +/// +/// The registry already holds names, required/optional arguments, types, enum +/// constraints, and hints; before this, the model saw that as a flat +/// `argumentTypes` map of six coarse cases and a prose `resultSummary`. Models +/// write markedly better code against real declarations, so the same metadata is +/// emitted as `.d.ts` — per capability for search results, and whole-surface for +/// a system prompt. +/// +/// Result types are `CodeModeValue` for now: built-in tools return untyped +/// `JSONValue` dictionaries, so a typed result would be a fiction. The prose +/// `resultSummary` is carried in the doc comment instead, and a per-capability +/// result schema can tighten this later without changing the shape here. +public enum TypeScriptDeclarations { + /// Shared types and the runtime globals that are not catalog capabilities. + /// + /// Hand-authored deliberately: these are the Node-compatibility shims in + /// `RuntimeJavaScript.bootstrap`, and unlike the namespaced helpers they take + /// *positional* arguments, which the catalog has no way to express. Keeping + /// them here means the declarations tell the truth about both conventions. + public static let preamble = """ + /** Any JSON-representable value. Bridge results are JSON, not class instances. */ + type CodeModeValue = + | null + | boolean + | number + | string + | CodeModeValue[] + | { [key: string]: CodeModeValue }; + + declare const console: { + log(...values: unknown[]): void; + info(...values: unknown[]): void; + warn(...values: unknown[]): void; + error(...values: unknown[]): void; + }; + + interface CodeModeResponse { + ok: boolean; + status: number; + statusText: string; + headers: { get(name: string): string | null }; + text(): Promise; + json(): Promise; + } + + /** + * HTTP(S) fetch. Runs on an isolated session: no app cookies, no stored + * credentials. Cookie/Authorization/Proxy-* request headers are refused + * unless the host app permits them. + */ + declare function fetch( + url: string, + options?: { + method?: string; + headers?: { [name: string]: string }; + body?: string; + bodyBase64?: string; + timeoutMs?: number; + responseEncoding?: "text" | "base64"; + } + ): Promise; + + /** + * Node-style filesystem aliases. Unlike the apple.* helpers these take + * POSITIONAL arguments and return Node-like values. + */ + declare const fs: { + promises: { + readFile(path: string, encoding?: "utf8" | "base64"): Promise; + writeFile(path: string, data: string, encoding?: "utf8" | "base64"): Promise; + readdir(path: string): Promise; + stat(path: string): Promise; + access(path: string): Promise; + mkdir(path: string, options?: { recursive?: boolean }): Promise; + rm(path: string, options?: { recursive?: boolean }): Promise; + rename(from: string, to: string, options?: { overwrite?: boolean; recursive?: boolean }): Promise; + copyFile(from: string, to: string, options?: { overwrite?: boolean; recursive?: boolean }): Promise; + }; + }; + + declare const path: { + join(...parts: string[]): string; + }; + """ + + /// The declaration for one capability, suitable for a search result payload. + public static func declaration(for reference: JavaScriptAPIReference) -> String { + guard let canonical = canonicalName(for: reference) else { + return "" + } + let name = canonical.replacingOccurrences(of: ".", with: "_") + return "\(documentation(for: reference, canonicalName: canonical))\ndeclare function \(name)\(functionSignature(for: reference));" + } + + /// The whole surface: the preamble plus every capability, grouped into + /// namespaces by its canonical dotted JavaScript name. + public static func surface(for references: [JavaScriptAPIReference]) -> String { + let root = Namespace(name: "") + for reference in references.sorted(by: { $0.capability < $1.capability }) { + guard let canonical = canonicalName(for: reference) else { + continue + } + var components = canonical.split(separator: ".").map(String.init) + guard components.count >= 2 else { + // A bare global such as `fetch` is covered by the preamble. + continue + } + let functionName = components.removeLast() + root.insert( + Member( + name: functionName, + documentation: documentation(for: reference, canonicalName: canonical), + signature: functionSignature(for: reference) + ), + at: components + ) + } + + let body = root.children + .sorted { $0.key < $1.key } + .map { render(namespace: $0.value, indent: "", isTopLevel: true) } + .joined(separator: "\n\n") + + return """ + // CodeMode JavaScript API — generated from the capability registry. + // Only helpers supported on the current host platform appear here. + + \(preamble) + + \(body) + """ + } + + // MARK: - Declaration pieces + + /// The namespaced helper name, preferred over Node-style aliases: aliases use + /// positional arguments the catalog cannot describe, and are documented in + /// the preamble instead. + private static func canonicalName(for reference: JavaScriptAPIReference) -> String? { + reference.jsNames.first { $0.contains(".") && $0.hasPrefix("fs.promises.") == false } + ?? reference.jsNames.first + } + + private static func documentation(for reference: JavaScriptAPIReference, canonicalName: String) -> String { + var lines = ["/**"] + lines.append(" * \(reference.summary)") + lines.append(" *") + lines.append(" * Capability: \(reference.capability)") + lines.append(" * Returns: \(reference.resultSummary)") + + let aliases = reference.jsNames.filter { $0 != canonicalName } + if aliases.isEmpty == false { + // Only the fs.promises.* shims are positional; other aliases are + // additional namespaced spellings of the same object-argument call. + let positional = aliases.filter { $0.hasPrefix("fs.promises.") || $0 == "fetch" } + let note = positional.isEmpty ? "" : " — \(positional.joined(separator: ", ")) take positional arguments" + lines.append(" * Aliases: \(aliases.joined(separator: ", "))\(note)") + } + if reference.example.isEmpty == false { + lines.append(" *") + lines.append(" * @example \(reference.example)") + } + lines.append(" */") + return lines.joined(separator: "\n") + } + + private static func functionSignature(for reference: JavaScriptAPIReference) -> String { + let tree = ArgumentTree(reference: reference) + guard tree.isEmpty == false else { + return "(): Promise" + } + // `args` is optional only when nothing in it is required. + let optionalMarker = reference.requiredArguments.isEmpty ? "?" : "" + return "(args\(optionalMarker): \(tree.render(indent: ""))): Promise" + } + + // MARK: - Argument tree + // + // Arguments are declared as dotted paths (`options.timeoutMs`), so the object + // literal has to be rebuilt from them. + + private struct ArgumentTree { + private var children: [String: ArgumentTree] = [:] + private var leaves: [Leaf] = [] + + private struct Leaf { + var name: String + var type: String + var optional: Bool + var hint: String? + } + + init(reference: JavaScriptAPIReference) { + let allPaths = Array(Set( + reference.requiredArguments + reference.optionalArguments + Array(reference.argumentTypes.keys) + )).sorted() + let required = Set(reference.requiredArguments) + + for path in allPaths { + let components = path.split(separator: ".").map(String.init) + insert( + components: components, + leaf: Leaf( + name: components.last ?? path, + type: TypeScriptDeclarations.tsType( + for: reference.argumentTypes[path] ?? .any, + allowedValues: reference.argumentConstraints.allowedStringValues[path] + ), + // A nested path is optional unless the exact path is + // listed as required. + optional: required.contains(path) == false, + hint: reference.argumentHints[path] + ) + ) + } + } + + private init() {} + + var isEmpty: Bool { + children.isEmpty && leaves.isEmpty + } + + private mutating func insert(components: [String], leaf: Leaf) { + guard components.count > 1 else { + // A declared parent (`options: object`) whose children also appear + // is represented by the nested literal, not by a duplicate leaf. + if children[leaf.name] == nil, leaves.contains(where: { $0.name == leaf.name }) == false { + leaves.append(leaf) + } + return + } + let head = components[0] + leaves.removeAll { $0.name == head } + var child = children[head] ?? ArgumentTree() + child.insert(components: Array(components.dropFirst()), leaf: leaf) + children[head] = child + } + + func render(indent: String) -> String { + let inner = indent + " " + var lines: [String] = ["{"] + + for leaf in leaves.sorted(by: { $0.name < $1.name }) { + if let hint = leaf.hint, hint.isEmpty == false { + lines.append("\(inner)/** \(hint) */") + } + lines.append("\(inner)\(leaf.name)\(leaf.optional ? "?" : ""): \(leaf.type);") + } + + for (name, child) in children.sorted(by: { $0.key < $1.key }) { + lines.append("\(inner)\(name)?: \(child.render(indent: inner));") + } + + lines.append("\(indent)}") + return lines.joined(separator: "\n") + } + } + + private static func tsType(for type: CapabilityArgumentType, allowedValues: [String]?) -> String { + if let allowedValues, allowedValues.isEmpty == false { + // The constraint is the type: a union beats `string` plus prose the + // model has to notice. + return allowedValues.map { "\"\($0)\"" }.joined(separator: " | ") + } + + switch type { + case .string: + return "string" + case .number: + return "number" + case .bool: + return "boolean" + case .object: + return "{ [key: string]: CodeModeValue }" + case .array: + return "CodeModeValue[]" + case .any: + return "CodeModeValue" + } + } + + // MARK: - Namespace rendering + + private struct Member { + var name: String + var documentation: String + var signature: String + } + + private final class Namespace { + let name: String + var children: [String: Namespace] = [:] + var members: [Member] = [] + + init(name: String) { + self.name = name + } + + func insert(_ member: Member, at path: [String]) { + guard let head = path.first else { + members.append(member) + return + } + let child = children[head] ?? Namespace(name: head) + children[head] = child + child.insert(member, at: Array(path.dropFirst())) + } + } + + private static func render(namespace: Namespace, indent: String, isTopLevel: Bool) -> String { + let inner = indent + " " + var lines = ["\(indent)\(isTopLevel ? "declare namespace" : "namespace") \(namespace.name) {"] + + for member in namespace.members.sorted(by: { $0.name < $1.name }) { + lines.append(indented(member.documentation, by: inner)) + lines.append("\(inner)function \(member.name)\(indented(member.signature, by: inner, skipFirstLine: true));") + } + + for (_, child) in namespace.children.sorted(by: { $0.key < $1.key }) { + lines.append(render(namespace: child, indent: inner, isTopLevel: false)) + } + + lines.append("\(indent)}") + return lines.joined(separator: "\n") + } + + private static func indented(_ text: String, by indent: String, skipFirstLine: Bool = false) -> String { + text + .split(separator: "\n", omittingEmptySubsequences: false) + .enumerated() + .map { index, line in + index == 0 && skipFirstLine ? String(line) : indent + line + } + .joined(separator: "\n") + } +} diff --git a/Sources/CodeMode/Host/CodeModeAgentGuidance.swift b/Sources/CodeMode/Host/CodeModeAgentGuidance.swift new file mode 100644 index 0000000..af596b2 --- /dev/null +++ b/Sources/CodeMode/Host/CodeModeAgentGuidance.swift @@ -0,0 +1,194 @@ +import Foundation + +/// System-prompt guidance that teaches an agent *why* it is writing a program +/// rather than calling tools one at a time. +/// +/// The tool descriptions and the generated TypeScript answer "what is this helper +/// called and what does it take". Neither answers "what shape should my work +/// take", and an agent that never learns the second question will use +/// `executeJavaScript` as a thin RPC — one call per operation — which is the +/// pattern this package exists to replace. +/// +/// Pair it with `CodeModeAgentTools.typeDeclarations()`, which supplies the +/// vocabulary this supplies the strategy for: +/// +/// ```swift +/// let prompt = CodeModeAgentGuidance.systemPrompt(.standard) +/// + "\n\n" +/// + tools.typeDeclarations() +/// ``` +public enum CodeModeAgentGuidance { + /// How much prompt budget to spend on the guidance. + /// + /// Every tier is self-contained and states the core idea; the larger ones add + /// worked examples and failure modes rather than qualifying what came before, + /// so a host can drop down a tier without losing a rule. + public enum Length: String, Sendable, CaseIterable, Comparable { + /// One paragraph: the core idea and nothing else. + case brief + /// The idea, the workflow, and one worked example. The default. + case standard + /// Adds fan-out and error-handling examples plus the anti-pattern list. + /// Worth it when the agent will do open-ended multi-step work. + case full + + public static func < (lhs: Length, rhs: Length) -> Bool { + lhs.order < rhs.order + } + + fileprivate var order: Int { + switch self { + case .brief: return 0 + case .standard: return 1 + case .full: return 2 + } + } + + /// Rough size of this tier's text, for budgeting against a prompt. + /// + /// Estimated at four characters per token — the usual English-prose rule + /// of thumb, not a real tokenization. Treat it as a bound to plan with, + /// not a number to assert on. + public var approximateTokenCount: Int { + CodeModeAgentGuidance.systemPrompt(self).count / 4 + } + } + + /// The guidance text at the requested length. + public static func systemPrompt(_ length: Length = .standard) -> String { + switch length { + case .brief: + return core + case .standard: + return [core, workflow, workedExample].joined(separator: "\n\n") + case .full: + return [core, workflow, workedExample, fanOutExample, antiPatterns].joined(separator: "\n\n") + } + } + + /// The largest tier that fits the budget, or `.brief` when nothing does. + /// + /// `.brief` is returned even when it exceeds the budget: some guidance is the + /// difference between an agent that batches and one that does not, so the + /// floor is deliberate rather than an empty string. + public static func systemPrompt(approximateTokenBudget: Int) -> String { + systemPrompt(length(forApproximateTokenBudget: approximateTokenBudget)) + } + + /// The tier `systemPrompt(approximateTokenBudget:)` would choose. + public static func length(forApproximateTokenBudget budget: Int) -> Length { + Length.allCases + .sorted(by: >) + .first { $0.approximateTokenCount <= budget } + ?? .brief + } + + // MARK: - Sections + + private static let core = """ + ## Writing code instead of calling tools + + You have a JavaScript runtime with direct access to this device's APIs. Use it \ + to run whole tasks, not single operations. One `executeJavaScript` call should \ + do the entire job — read, loop, branch, filter, transform, write — and return \ + only the finished answer. Ten helper calls inside one script is normal and \ + fast; ten `executeJavaScript` calls that each make one helper call is the \ + mistake this runtime exists to avoid, because every round trip costs a model \ + turn and pushes intermediate data you do not need through your context. \ + Request only the capabilities your script actually calls. + """ + + private static let workflow = """ + ## How to work + + 1. Call `searchJavaScriptAPI` first when you are unsure of helper names, \ + arguments, or result shapes. Return `ref.dts` — the TypeScript declaration — \ + rather than assembling the metadata fields yourself. + 2. Write one script that completes the task. Reach for ordinary control flow: \ + loops over collections, `if` for branching, `try`/`catch` around anything that \ + may fail per item. + 3. Do the filtering and aggregation *in the script*. Return the graded answer — \ + a count, a summary, the three matching records — not the raw collection you \ + read. A large return value is truncated, and the data you did not need cost \ + you context on the way through. + 4. Repair from the structured error. It names the capability, the line and \ + column, and what to change; the suggestions tell you when a retry cannot help. + """ + + private static let workedExample = """ + ## The shape to aim for + + Task: "how much did I spend on the trip, and file the receipts?" + + ```javascript + // ONE call. Reads, filters, sums, writes, and returns just the total. + const entries = await apple.fs.list({ path: 'documents:receipts' }); + let total = 0; + const filed = []; + + for (const entry of entries) { + if (entry.isDirectory || !entry.name.endsWith('.json')) continue; + const { text } = await apple.fs.read({ path: entry.path }); + const receipt = JSON.parse(text); + if (receipt.trip !== 'lisbon') continue; + total += receipt.amount; + await apple.fs.move({ + from: entry.path, + to: `documents:receipts/lisbon/${entry.name}`, + overwrite: true + }); + filed.push(entry.name); + } + + return { total, filedCount: filed.length }; + ``` + + `allowedCapabilities: ["fs.list", "fs.read", "fs.move"]` — exactly the three \ + helpers the script calls, and nothing else. + """ + + private static let fanOutExample = """ + ## Handling many items and partial failure + + Keep going when one item fails, and report what happened rather than throwing \ + the whole run away: + + ```javascript + const results = []; + for (const city of ['lisbon', 'porto', 'faro']) { + try { + const response = await fetch(`https://api.example.com/weather/${city}`); + if (!response.ok) { results.push({ city, error: response.status }); continue; } + const data = await response.json(); + results.push({ city, tempC: data.current.temp_c }); + } catch (error) { + results.push({ city, error: String(error) }); + } + } + return results; + ``` + + Native calls run one at a time, so `Promise.all` does not make these overlap — \ + a loop is just as fast here and reads better. Use \ + `await new Promise(r => setTimeout(r, 1000))` to back off between retries; \ + delays are real. + """ + + private static let antiPatterns = """ + ## Anti-patterns + + - **One call per operation.** If your second `executeJavaScript` starts from \ + data the first one returned, both should have been one script. + - **Returning the raw collection.** Reading 500 records to answer "how many \ + are overdue" should return a number, not 500 records. + - **Asking for capabilities you do not call.** Request the exact set your \ + script uses; a wider list is refused by hosts that enforce a ceiling, and it \ + is the wrong thing to ask a user to approve. + - **Forgetting `await`.** An un-awaited helper's failure is silently \ + discarded and you will be told the run succeeded. A \ + `BRIDGE_FAILURES_NOT_SURFACED` diagnostic means exactly this happened. + - **Re-running a call the error told you not to retry.** \ + `NETWORK_POLICY_VIOLATION`, `UI_PRESENTER_UNAVAILABLE`, and a host-withheld \ + `CAPABILITY_DENIED` will not succeed on a second attempt. + """ +} diff --git a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift index 62cace0..7585c47 100644 --- a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift +++ b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift @@ -42,7 +42,7 @@ public enum CodeModeAgentToolDescriptions { "items": .object([ "type": .string("string"), ]), - "description": .string("Custom provider capability keys required by the JavaScript. Built-in capability IDs may also be accepted here by hosts that expose one allowlist field."), + "description": .string("Custom provider capability keys required by the JavaScript, for example myapp.tasks.complete. Built-in capability IDs do not belong here and are ignored — list those in allowedCapabilities."), ]), "timeoutMs": .object([ "type": .string("integer"), @@ -75,6 +75,7 @@ public enum CodeModeAgentToolDescriptions { argumentHints: Record; argumentConstraints: { allowedStringValues: Record }; resultSummary: string; + dts: string; } declare const api: { @@ -86,6 +87,8 @@ public enum CodeModeAgentToolDescriptions { Your code must evaluate to an async function and return JSON-serializable output. Search has a 2s budget, so slice broad result sets before returning them. + Prefer returning ref.dts: it is the TypeScript declaration for the helper, with argument names, types, string-literal unions for constrained values, optionality, and the result summary. It is more precise and usually shorter than assembling argumentTypes, argumentHints, and argumentConstraints yourself. + Examples: async () => { return api.references @@ -106,7 +109,14 @@ public enum CodeModeAgentToolDescriptions { } async () => { - return api.byJSName["apple.fs.read"]; + return api.byJSName["apple.fs.read"].dts; + } + + async () => { + return api.references + .filter(ref => ref.tags.includes("calendar")) + .map(ref => ref.dts) + .join("\n\n"); } """ ) @@ -114,21 +124,56 @@ public enum CodeModeAgentToolDescriptions { public static let executeJavaScript = CodeModeAgentToolDescription( name: "executeJavaScript", description: """ - Execute JavaScript against the CodeMode runtime. Prefer searchJavaScriptAPI first when choosing helpers or arguments. Cross-platform helpers live under apple.* and platform-specific helpers live under platform namespaces such as ios.alarm.*; custom host providers may expose additional namespaces. System UI helpers such as apple.ui.presentAlert, apple.calendar.presentNewEvent, apple.photos.pick, apple.contacts.pick, apple.documents.pick, apple.share.present, apple.quicklook.preview, apple.web.present, and apple.auth.webAuthenticate require a host-provided SystemUIPresenter; camera, document scan, mail, and message compose helpers are iOS-only. Only helpers supported on the current host platform are installed. + Execute JavaScript against the CodeMode runtime. Use one call to run a whole task — read, loop, branch, filter, transform, write — and return only the finished answer. Many helper calls inside one script is the intended shape; many executeJavaScript calls that each make one helper call is not, because every round trip costs a turn and pushes intermediate data through your context. Prefer searchJavaScriptAPI first when choosing helpers or arguments. Cross-platform helpers live under apple.* and platform-specific helpers live under platform namespaces such as ios.alarm.*; custom host providers may expose additional namespaces. System UI helpers such as apple.ui.presentAlert, apple.calendar.presentNewEvent, apple.photos.pick, apple.contacts.pick, apple.documents.pick, apple.share.present, apple.quicklook.preview, apple.web.present, and apple.auth.webAuthenticate require a host-provided SystemUIPresenter; camera, document scan, mail, and message compose helpers are iOS-only. Only helpers supported on the current host platform are installed. + + Calling conventions: apple.* and other namespaced helpers take one object argument, for example apple.fs.read({ path: "tmp:file.txt" }) and often return structured objects such as { text, base64 }. Node-style filesystem aliases under fs.promises.* use positional arguments, for example fs.promises.readFile("tmp:file.txt", "utf8"), and return Node-like values such as a string for readFile. When in doubt, use the ref.dts from searchJavaScriptAPI — it states the exact signature. fetch(url, options) is a global helper and returns a Response-like object with text(), json(), headers.get(), status, and ok. + + Return semantics: the runtime wraps your code in an async function. For multi-statement code, return the final graded value with an explicit top-level return statement. A script that is only a bare final top-level await expression, such as await apple.fs.read({ path: "tmp:file.txt" }), returns that awaited value. Do not use an unreturned async IIFE as the final expression. - Calling conventions: apple.* helpers take one object argument, for example apple.fs.read({ path: "tmp:file.txt" }) and often return structured objects such as { text, base64 }. Node-style filesystem aliases use positional arguments, for example fs.promises.readFile("tmp:file.txt", "utf8"), and return Node-like values such as a string for readFile. fetch(url, options) is a global helper and returns a Response-like object with text(), json(), headers.get(), status, and ok. + Examples. Do the whole job in one script and return the graded answer, not the raw data you read: - Return semantics: the runtime wraps your code in an async function. For multi-statement code, return the final graded value with an explicit top-level return statement. A script that is only a bare final top-level await expression, such as await apple.fs.read({ path: "tmp:file.txt" }), returns that awaited value. Do not use an unreturned async IIFE as the final expression. setTimeout callbacks fire synchronously in this runtime. + const start = new Date().toISOString(); + const end = new Date(Date.now() + 7 * 86400000).toISOString(); + const events = await apple.calendar.listEvents({ start, end }); + const conflicts = events.filter((event, index) => + events.slice(index + 1).some(other => other.startDate < event.endDate) + ); + return { total: events.length, conflicts: conflicts.map(event => event.title) }; - Allowlisting: include only the required built-in capabilities in allowedCapabilities and custom provider keys in allowedCapabilityKeys. Execution defaults to a 10000ms timeout and returns structured CodeModeToolError failures for syntax errors, missing JS helpers, runtime throws, validation failures, permission denials, timeouts, cancellation, and internal errors. + Prefer one call that takes a collection over a loop of single calls — check the catalog for a plural argument before writing the loop: + + const contacts = await apple.contacts.list({ identifiers: ids }); + + When the work genuinely is per-item, keep going after a failure and report what happened: + + const results = []; + for (const city of cities) { + try { + const response = await fetch(`https://api.example.com/weather/${city}`); + results.push({ city, ok: response.ok, data: response.ok ? await response.json() : null }); + } catch (error) { + results.push({ city, error: String(error) }); + } + } + return results; + + Concurrency: setTimeout and clearTimeout work normally — callbacks are deferred and delays are honoured, so `await new Promise(r => setTimeout(r, 1000))` really waits. There is no I/O event loop, though: native calls run one at a time, so Promise.all over several helpers completes them sequentially rather than concurrently. A promise with no resolve path and no pending timer can never settle and fails fast with JS_RUNTIME_ERROR rather than running out the clock. + + Allowlisting: request the exact set of capabilities your script calls and nothing more — a capability you list but never use is a wider grant than the task needs, and hosts that enforce a ceiling will refuse it. Built-in capability IDs go in allowedCapabilities; custom provider keys go in allowedCapabilityKeys. The two fields are not interchangeable: a built-in capability listed in allowedCapabilityKeys is ignored. These fields declare what your script needs; the host app applies its own ceiling on top, so a capability you list may still be withheld. Execution defaults to a 10000ms timeout and returns structured CodeModeToolError failures for syntax errors, missing JS helpers, runtime throws, validation failures, permission denials, timeouts, cancellation, and internal errors. Error repair guide: JS_API_NOT_FOUND: use the suggested JS helper names or searchJavaScriptAPI. - CAPABILITY_DENIED: add the exact capability to allowedCapabilities or allowedCapabilityKeys and retry. + CAPABILITY_DENIED: add the exact capability to allowedCapabilities (built-ins) or allowedCapabilityKeys (custom provider keys) and retry — unless the error suggestions say the host withheld it, in which case retrying will not help. PERMISSION_DENIED: the capability is allowlisted but the OS, host, or custom provider denied permission; request permission if a helper exists, otherwise tell the user or host. UI_PRESENTER_UNAVAILABLE: host configuration issue; do not retry the same call. INVALID_ARGUMENTS: use the catalog requiredArguments, optionalArguments, argumentHints, and example. NETWORK_POLICY_VIOLATION: the host's network access policy refused the destination; do not retry the same URL and do not add capabilities to work around it. + JS_RUNTIME_ERROR "can never settle": a promise in your script has no resolve path; await every promise and make sure each one resolves. + + Warning diagnostics worth acting on: + BRIDGE_FAILURES_NOT_SURFACED: a helper failed but your script still returned successfully — usually a missing await, which discards the rejection. + RESULT_TRUNCATED: the return value exceeded the size limit; return a summary or write the payload to the sandbox and return the path. + TIMER_CALLBACK_ERROR: a setTimeout callback threw; its error cannot reach the caller, so handle it inside the callback. """ ) diff --git a/Sources/CodeMode/Host/CodeModeAgentTools.swift b/Sources/CodeMode/Host/CodeModeAgentTools.swift index 1959dfb..03e91a8 100644 --- a/Sources/CodeMode/Host/CodeModeAgentTools.swift +++ b/Sources/CodeMode/Host/CodeModeAgentTools.swift @@ -8,6 +8,7 @@ public final class CodeModeAgentTools: @unchecked Sendable { public init(config: CodeModeConfiguration = .init()) { let allDefaultRegistrations = DefaultCapabilityLoader.loadAllRegistrations( fileSystem: config.fileSystem, + fileSystemLimits: config.fileSystemLimits, networkAccessPolicy: config.networkAccessPolicy, eventInbox: config.eventInbox, cloudKitClient: config.cloudKitClient, @@ -45,7 +46,43 @@ public final class CodeModeAgentTools: @unchecked Sendable { try await runtime.searchAsync(request) } + /// Declared `async throws` for a body that is currently synchronous and + /// non-throwing. That is deliberate: making the bridge ABI asynchronous is + /// planned, and this signature is the one that will not have to change. public func executeJavaScript(_ request: JavaScriptExecutionRequest) async throws -> JavaScriptExecutionCall { runtime.makeExecutionCall(request) } + + /// TypeScript declarations for the whole platform-filtered API surface. + /// + /// Models write markedly better code against real types than against prose, + /// so a host can drop this into its system prompt when the surface is small + /// enough to afford, and rely on `searchJavaScriptAPI` (whose results carry a + /// per-capability `dts`) when it is not. + public func typeDeclarations() -> String { + catalog.typeDeclarations() + } + + /// Every capability available on this host, for consent UI and for hosts + /// building their own `CapabilityGrant`. + public func capabilities() -> [JavaScriptAPIReference] { + catalog.allReferences() + } + + /// Adds host providers after construction. + /// + /// Providers no longer have to be known at init: the catalog tracks the + /// registry, so search starts advertising these immediately and the JS + /// bindings are installed for the next execution. A host whose domain APIs + /// depend on runtime state — a signed-in account, a loaded document — can + /// register them when that state arrives. + /// + /// Registering a key that already exists replaces it. + public func register(providers: [any CodeModeProvider]) { + registry.register(providers.flatMap { $0.codeModeRegistrations() }) + } + + public func register(provider: any CodeModeProvider) { + register(providers: [provider]) + } } diff --git a/Sources/CodeMode/Registry/CapabilityRegistry.swift b/Sources/CodeMode/Registry/CapabilityRegistry.swift index b50f76d..e5bb049 100644 --- a/Sources/CodeMode/Registry/CapabilityRegistry.swift +++ b/Sources/CodeMode/Registry/CapabilityRegistry.swift @@ -16,7 +16,10 @@ public enum CapabilityArgumentType: String, Sendable, Codable, Equatable { if case .string = value { return true } return false case .number: - if case .number = value { return true } + // Non-finite is not a usable number for any bridge: it cannot be + // converted to an Int, cannot round-trip through JSON, and reaches + // here from `"inf"`/`"nan"` coercion or a JS `Infinity`. + if case let .number(number) = value { return number.isFinite } return false case .bool: if case .bool = value { return true } @@ -432,6 +435,11 @@ public final class CapabilityRegistry: @unchecked Sendable { private let lock = NSLock() private var registrations: [CapabilityID: CapabilityRegistration] = [:] private var codeModeRegistrations: [CodeModeCapabilityKey: CodeModeRegistration] = [:] + /// Bumped on every mutation so `BridgeCatalog` can tell whether its snapshot + /// is still current. Without it the catalog was frozen at init, which made + /// the public `register(...)` methods below either unreachable or a way to + /// desync what search advertises from what execution can actually invoke. + private var generationValue = 0 public init(registrations: [CapabilityRegistration] = [], codeModeRegistrations: [CodeModeRegistration] = []) { for registration in registrations { @@ -442,9 +450,17 @@ public final class CapabilityRegistry: @unchecked Sendable { } } + /// Monotonic version of the registry's contents. + public var generation: Int { + lock.lock() + defer { lock.unlock() } + return generationValue + } + public func register(_ registration: CapabilityRegistration) { lock.lock() registrations[registration.descriptor.id] = registration + generationValue += 1 lock.unlock() } @@ -453,12 +469,14 @@ public final class CapabilityRegistry: @unchecked Sendable { for registration in registrations { self.registrations[registration.descriptor.id] = registration } + generationValue += 1 lock.unlock() } public func register(_ registration: CodeModeRegistration) { lock.lock() codeModeRegistrations[registration.capabilityKey] = registration + generationValue += 1 lock.unlock() } @@ -467,6 +485,7 @@ public final class CapabilityRegistry: @unchecked Sendable { for registration in registrations { self.codeModeRegistrations[registration.capabilityKey] = registration } + generationValue += 1 lock.unlock() } @@ -506,14 +525,13 @@ public final class CapabilityRegistry: @unchecked Sendable { return registrations[capability].map(RegisteredCodeModeFunction.init) } + /// Resolves a *custom provider* key only. Built-ins are deliberately not + /// reachable here: they are dispatched by `CapabilityID` and gated by + /// `allowedCapabilities`, so resolving them through the loosely-typed key + /// namespace would let `allowedCapabilityKeys` alias a built-in. func registeredFunction(for capabilityKey: CodeModeCapabilityKey) -> RegisteredCodeModeFunction? { lock.lock() defer { lock.unlock() } - if let builtIn = CapabilityID(rawValue: capabilityKey.rawValue), - let registration = registrations[builtIn] - { - return RegisteredCodeModeFunction(registration) - } return codeModeRegistrations[capabilityKey].map(RegisteredCodeModeFunction.init) } @@ -535,7 +553,11 @@ public final class CapabilityRegistry: @unchecked Sendable { } private func invokeBuiltIn(_ capability: CapabilityID, arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { - guard context.allowedCapabilities.contains(capability) || context.allowedCapabilityKeys.contains(capability.codeModeKey) else { + // Built-ins are gated by `allowedCapabilities` alone. `allowedCapabilityKeys` + // accepts arbitrary strings and is not validated against `CapabilityID` at + // decode time, so honouring it here would bypass any host that vets only + // the strictly-typed field. + guard context.allowedCapabilities.contains(capability) else { throw BridgeError.capabilityDenied(capability) } @@ -547,6 +569,13 @@ public final class CapabilityRegistry: @unchecked Sendable { } private func invokeCodeMode(_ capabilityKey: CodeModeCapabilityKey, arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { + // Defense in depth: `invoke` already routes anything that parses as a + // built-in to `invokeBuiltIn`, but a key that spells a built-in must never + // reach a handler through the provider path. + if let builtIn = CapabilityID(rawValue: capabilityKey.rawValue) { + throw BridgeError.capabilityDenied(builtIn) + } + guard context.allowedCapabilityKeys.contains(capabilityKey) else { throw BridgeError.capabilityKeyDenied(capabilityKey) } @@ -594,7 +623,10 @@ public final class CapabilityRegistry: @unchecked Sendable { let trimmed = raw.trimmingCharacters(in: .whitespaces) switch type { case .number: - guard let number = Double(trimmed) else { return nil } + // `Double.init(String)` accepts "inf", "-infinity", and "nan", so a + // declared-number argument sent as a string is a way to smuggle a + // non-finite value past the type check. + guard let number = Double(trimmed), number.isFinite else { return nil } return .number(number) case .bool: switch trimmed.lowercased() { diff --git a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift index 4a64acf..1dc9cc2 100644 --- a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift +++ b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift @@ -2,8 +2,17 @@ import Foundation public final class BridgeInvocationContext: @unchecked Sendable { public let executionContext: ExecutionContext + /// Built-in capabilities this execution may reach: the model's declaration + /// intersected with the host's `CapabilityGrant`. public let allowedCapabilities: Set + /// Custom provider keys this execution may reach. Deliberately disjoint from + /// `allowedCapabilities` — a built-in capability is never granted by a key, or + /// `allowedCapabilityKeys` would be a way around a host that vets only the + /// strictly-typed `allowedCapabilities`. public let allowedCapabilityKeys: Set + /// Identifiers the request asked for and the host's grant withheld. Used to + /// tell the model a denial is not repairable by widening `allowedCapabilities`. + public let hostWithheldCapabilityIdentifiers: Set public let pathPolicy: any PathPolicy public let artifactStore: any ArtifactStore public let permissionBroker: any PermissionBroker @@ -12,6 +21,7 @@ public final class BridgeInvocationContext: @unchecked Sendable { private let lock = NSLock() private var validatedPermissions: Set = [] + private var failedCapabilityInvocations: [(capability: String, code: String)] = [] private let transcript: ExecutionTranscript private let cancellationController: ExecutionCancellationController @@ -19,6 +29,7 @@ public final class BridgeInvocationContext: @unchecked Sendable { executionContext: ExecutionContext, allowedCapabilities: Set, allowedCapabilityKeys: Set = [], + hostWithheldCapabilityIdentifiers: Set = [], pathPolicy: any PathPolicy, artifactStore: any ArtifactStore, permissionBroker: any PermissionBroker, @@ -29,7 +40,8 @@ public final class BridgeInvocationContext: @unchecked Sendable { ) { self.executionContext = executionContext self.allowedCapabilities = allowedCapabilities - self.allowedCapabilityKeys = allowedCapabilityKeys.union(allowedCapabilities.map(\.codeModeKey)) + self.allowedCapabilityKeys = allowedCapabilityKeys + self.hostWithheldCapabilityIdentifiers = hostWithheldCapabilityIdentifiers self.pathPolicy = pathPolicy self.artifactStore = artifactStore self.permissionBroker = permissionBroker @@ -97,12 +109,64 @@ public final class BridgeInvocationContext: @unchecked Sendable { return resolved } + /// True when the request declared this identifier but the host's + /// `CapabilityGrant` withheld it — a denial the model cannot repair. + func isWithheldByHost(_ identifier: String) -> Bool { + hostWithheldCapabilityIdentifiers.contains(identifier) + } + + /// Builds a `CodeModeToolError` carrying this execution's transcript. + /// + /// Lives here rather than on the runtime because the transcript fields are + /// this type's; three separate call sites were otherwise assembling the same + /// three accessors by hand. + func toolError( + code: String, + message: String, + functionName: String? = nil, + capability: CapabilityID? = nil, + capabilityKey: CodeModeCapabilityKey? = nil, + suggestions: [String] = [] + ) -> CodeModeToolError { + CodeModeToolError( + code: code, + message: message, + functionName: functionName, + capability: capability, + capabilityKey: capabilityKey, + suggestions: suggestions, + diagnostics: allDiagnostics(), + logs: allLogs(), + permissionEvents: allPermissionEvents() + ) + } + func recordDiagnostic(_ diagnostic: ToolDiagnostic) { transcript.record(diagnostic: diagnostic) } + /// Notes a capability call that failed, so a script that finishes + /// *successfully* despite failed bridge calls can be flagged. Nothing installs + /// an unhandled-rejection hook — a forgotten `await` is the most common LLM + /// JavaScript mistake, and it silently discards a `CAPABILITY_DENIED` while + /// the agent is told the run succeeded. + func recordCapabilityFailure(capability: String, code: String) { + lock.lock() + failedCapabilityInvocations.append((capability, code)) + lock.unlock() + } + + func capabilityFailures() -> [(capability: String, code: String)] { + lock.lock() + defer { lock.unlock() } + return failedCapabilityInvocations + } + func checkCancellation() throws { - if cancellationController.isCancelled || Task.isCancelled { + // Bridge handlers run on a GCD worker, not inside a Task, so + // `Task.isCancelled` was always false here. The controller is the signal + // `JavaScriptExecutionCall.cancel()` actually sets. + if cancellationController.isCancelled { throw BridgeError.cancelled } } diff --git a/Sources/CodeMode/Runtime/BridgeRuntime.swift b/Sources/CodeMode/Runtime/BridgeRuntime.swift index 4eec3c2..5691521 100644 --- a/Sources/CodeMode/Runtime/BridgeRuntime.swift +++ b/Sources/CodeMode/Runtime/BridgeRuntime.swift @@ -18,6 +18,11 @@ final class BridgeRuntime: @unchecked Sendable { var suggestions: [String] } + /// Ceiling on the re-armed watchdog budget for result serialization. Bounded + /// independently of the execution timeout so a long-running script cannot buy + /// an equally long serialization window. + private static let serializationBudgetMs = 1_000 + private let registry: CapabilityRegistry private let catalog: BridgeCatalog private let config: CodeModeConfiguration @@ -27,6 +32,14 @@ final class BridgeRuntime: @unchecked Sendable { qos: .userInitiated, attributes: .concurrent ) + /// Caps how many executions hold a thread and a live `JSContext` at once. + /// + /// The queue is concurrent and each execution blocks its thread for the whole + /// run, so thirty parallel `executeJavaScript` calls meant thirty blocked GCD + /// threads and thirty JavaScriptCore VMs. Excess executions now queue instead + /// of exhausting the thread pool; `timeoutMs` still measures the run itself, + /// not the wait for a slot. + private let executionSlots: DispatchSemaphore init( registry: CapabilityRegistry, @@ -38,6 +51,7 @@ final class BridgeRuntime: @unchecked Sendable { self.catalog = catalog self.config = config self.unsupportedBuiltInJavaScriptNames = unsupportedBuiltInJavaScriptNames + self.executionSlots = DispatchSemaphore(value: max(1, config.executionLimits.maxConcurrentExecutions)) } func search(_ request: JavaScriptAPISearchRequest) throws -> JavaScriptAPISearchResponse { @@ -49,7 +63,7 @@ final class BridgeRuntime: @unchecked Sendable { ) } - let transcript = ExecutionTranscript() + let transcript = ExecutionTranscript(limits: config.executionLimits) let cancellationController = ExecutionCancellationController() let invocationContext = BridgeInvocationContext( executionContext: .init(), @@ -104,10 +118,16 @@ final class BridgeRuntime: @unchecked Sendable { func makeExecutionCall(_ request: JavaScriptExecutionRequest) -> JavaScriptExecutionCall { let cancellationController = ExecutionCancellationController() let continuationBox = LockedBox.Continuation?>(nil) - let events = AsyncStream { continuation in + // Bounded: the default policy buffers without limit, so a chatty script + // whose events the host is not draining grows the buffer until the app + // dies. Dropping the oldest keeps the terminal events — which is what + // the host actually needs — and the full transcript is on the result. + let events = AsyncStream( + bufferingPolicy: .bufferingNewest(config.executionLimits.maxBufferedEvents) + ) { continuation in continuationBox.set(continuation) } - let transcript = ExecutionTranscript { event in + let transcript = ExecutionTranscript(limits: config.executionLimits) { event in continuationBox.get()?.yield(event) } @@ -168,6 +188,10 @@ final class BridgeRuntime: @unchecked Sendable { ) async throws -> Output { try await withCheckedThrowingContinuation { continuation in executionQueue.async { + // Wait for a slot on the worker, not before dispatching, so the + // caller's async context is never blocked. + self.executionSlots.wait() + defer { self.executionSlots.signal() } do { continuation.resume(returning: try operation()) } catch { @@ -182,10 +206,35 @@ final class BridgeRuntime: @unchecked Sendable { transcript: ExecutionTranscript, cancellationController: ExecutionCancellationController ) throws -> JavaScriptExecutionResult { + // `allowedCapabilities` / `allowedCapabilityKeys` are model-authored, so they + // are a declaration of intent, not a boundary. The host's grant is the + // boundary: the effective set is always requested ∩ granted. + let grant = config.capabilityGrant.resolve( + requestedCapabilities: Set(request.allowedCapabilities), + requestedCapabilityKeys: Set(request.allowedCapabilityKeys) + ) + + let withheld = grant.withheld + if withheld.isEmpty == false { + transcript.record( + diagnostic: ToolDiagnostic( + severity: .warning, + code: "CAPABILITY_WITHHELD_BY_HOST", + message: "The host's capability grant withheld: \(withheld.joined(separator: ", ")). Calls needing these fail with CAPABILITY_DENIED and cannot be repaired from the script.", + category: "security", + suggestions: [ + "Do not retry with a wider allowedCapabilities; the host decides this set.", + "Complete what you can with the granted capabilities, or report the missing access to the user.", + ] + ) + ) + } + let invocationContext = BridgeInvocationContext( executionContext: request.context, - allowedCapabilities: Set(request.allowedCapabilities), - allowedCapabilityKeys: Set(request.allowedCapabilityKeys), + allowedCapabilities: grant.capabilities, + allowedCapabilityKeys: grant.capabilityKeys, + hostWithheldCapabilityIdentifiers: Set(withheld), pathPolicy: config.pathPolicy, artifactStore: config.artifactStore, permissionBroker: config.permissionBroker, @@ -256,8 +305,10 @@ final class BridgeRuntime: @unchecked Sendable { let errorPayload = self.bridgeFailurePayload( for: bridgeError, capability: capabilityID, - capabilityKey: capabilityKey + capabilityKey: capabilityKey, + invocationContext: invocationContext ) + invocationContext.recordCapabilityFailure(capability: capability, code: errorPayload.code) invocationContext.log(.error, message: "Capability failed \(capability): \(errorPayload.message)") invocationContext.auditLogger.log(AuditEvent(capability: capability, message: "failed: \(errorPayload.message)")) @@ -445,16 +496,14 @@ final class BridgeRuntime: @unchecked Sendable { switch watchdog.termination { case .timedOut: - throw toolError( + throw invocationContext.toolError( code: "EXECUTION_TIMEOUT", - message: "Execution timed out after \(timeoutMs)ms", - transcript: invocationContext + message: "Execution timed out after \(timeoutMs)ms" ) case .cancelled: - throw toolError( + throw invocationContext.toolError( code: "CANCELLED", - message: "Execution cancelled", - transcript: invocationContext + message: "Execution cancelled" ) case nil: break @@ -467,22 +516,31 @@ final class BridgeRuntime: @unchecked Sendable { let settlement = try waitForSettlement( context: context, - watchdog: watchdog, - cancellationController: cancellationController, - invocationContext: invocationContext, - timeoutCode: "EXECUTION_TIMEOUT", - timeoutMessage: "Execution timed out after \(timeoutMs)ms", - cancelMessage: "Execution cancelled" + settlement: SettlementParameters( + watchdog: watchdog, + cancellationController: cancellationController, + invocationContext: invocationContext, + timeoutCode: "EXECUTION_TIMEOUT", + timeoutMessage: "Execution timed out after \(timeoutMs)ms", + cancelMessage: "Execution cancelled" + ) ) // Give result serialization its own bounded budget so a script that // settled near the deadline still serializes, while a runaway getter or - // toJSON is terminated instead of hanging the execution thread. - watchdog.rearm(timeoutMs: max(timeoutMs, 1_000)) + // toJSON is terminated instead of hanging the execution thread. `min`, + // not `max`: the point is a *bounded* budget, and `max` handed a 60s + // execution another 60s just to run JSON.stringify. + watchdog.rearm(timeoutMs: min(timeoutMs, Self.serializationBudgetMs)) switch settlement { case .fulfilled: - let output = try decodeOutput(from: context) + recordSwallowedFailures(invocationContext: invocationContext) + let output = try decodeOutput( + from: context, + watchdog: watchdog, + invocationContext: invocationContext + ) if output == nil { invocationContext.recordDiagnostic(Self.noReturnValueDiagnostic(for: code)) } @@ -498,25 +556,36 @@ final class BridgeRuntime: @unchecked Sendable { case rejected } - /// Polls the wrapped promise's settled state until it resolves, is - /// cancelled, or the watchdog deadline passes. The settled state is checked - /// before the deadline on every iteration, so a script that has already - /// fulfilled — the common case under the synchronous bridge model — returns - /// its result even if evaluation finished a hair past the deadline, instead - /// of being discarded with a spurious timeout. + /// Runs the runtime's event loop until the wrapped promise settles, the + /// deadline passes, or the program is provably stuck. + /// + /// The settled state is checked before the deadline on every iteration, so a + /// script that has already fulfilled returns its result even if evaluation + /// finished a hair past the deadline, instead of being discarded with a + /// spurious timeout. + /// + /// When the promise is still pending, the *only* thing that can advance it + /// under the synchronous bridge model is a queued timer — microtasks have + /// already drained inside `evaluateScript`, and no native call is + /// outstanding. So this either sleeps until the next timer is due and fires + /// it, or, when no timer is queued, reports immediately that the promise can + /// never settle instead of sleeping a thread to the deadline for nothing. private func waitForSettlement( context: JSContext, - watchdog: ExecutionWatchdog, - cancellationController: ExecutionCancellationController, - invocationContext: BridgeInvocationContext, - timeoutCode: String, - timeoutMessage: String, - cancelMessage: String + settlement: SettlementParameters ) throws -> SettlementState { while true { - if cancellationController.isCancelled || Task.isCancelled { - cancellationController.cancel() - throw toolError(code: "CANCELLED", message: cancelMessage, transcript: invocationContext) + // Cancellation only, deliberately: the deadline is checked *after* + // the settled state below, so a script that fulfilled a hair past it + // still returns its result instead of a spurious timeout. + // + // `Task.isCancelled` is not checked at all — this runs on a plain GCD + // worker with no surrounding Task, so it was always false, dead code + // that read as a second layer of cancellation support. The + // controller, which `JavaScriptExecutionCall.cancel()` sets, is the + // real signal. + if settlement.cancellationController.isCancelled { + throw settlement.error(code: "CANCELLED", message: settlement.cancelMessage) } let state = context.evaluateScript("globalThis.__codemode.state")?.toString() ?? "unknown" @@ -526,22 +595,134 @@ final class BridgeRuntime: @unchecked Sendable { case "rejected": return .rejected default: - if let termination = watchdog.termination { - switch termination { - case .cancelled: - throw toolError(code: "CANCELLED", message: cancelMessage, transcript: invocationContext) - case .timedOut: - throw toolError(code: timeoutCode, message: timeoutMessage, transcript: invocationContext) - } - } - if Date() >= watchdog.deadline { - throw toolError(code: timeoutCode, message: timeoutMessage, transcript: invocationContext) + try checkInterrupted(settlement) + + let nextTimerDelay = context + .evaluateScript("globalThis.__codemode.nextTimerDelay()")? + .toNumber()? + .doubleValue ?? -1 + + guard nextTimerDelay >= 0 else { + throw settlement.error( + code: "JS_RUNTIME_ERROR", + message: "Script finished with a promise that can never settle: nothing is left to resolve it. This runtime has no I/O event loop, so only a pending setTimeout can advance a program after the synchronous work is done.", + suggestions: [ + "Await every promise the script creates, and make sure each one has a resolve path.", + "A `new Promise(...)` whose executor never calls resolve or reject will hang forever here.", + ] + ) } - Thread.sleep(forTimeInterval: 0.01) + + try runDueTimers(delayMs: nextTimerDelay, context: context, settlement: settlement) } } } + /// Everything the settlement loop needs to report a cancellation or timeout. + /// The execution and search paths differ only in these message strings. + private struct SettlementParameters { + var watchdog: ExecutionWatchdog + var cancellationController: ExecutionCancellationController + var invocationContext: BridgeInvocationContext + var timeoutCode: String + var timeoutMessage: String + var cancelMessage: String + + func error(code: String, message: String, suggestions: [String] = []) -> CodeModeToolError { + invocationContext.toolError(code: code, message: message, suggestions: suggestions) + } + } + + /// Throws if the run was cancelled or has run out of time — the check every + /// waiting loop in this file has to make before sleeping again. + private func checkInterrupted(_ settlement: SettlementParameters) throws { + if settlement.cancellationController.isCancelled { + throw settlement.error(code: "CANCELLED", message: settlement.cancelMessage) + } + switch settlement.watchdog.termination { + case .cancelled: + throw settlement.error(code: "CANCELLED", message: settlement.cancelMessage) + case .timedOut: + throw settlement.error(code: settlement.timeoutCode, message: settlement.timeoutMessage) + case nil: + break + } + if settlement.watchdog.hasPassedDeadline { + throw settlement.error(code: settlement.timeoutCode, message: settlement.timeoutMessage) + } + } + + /// Sleeps out a timer's delay — bounded by the deadline and responsive to + /// cancellation — then fires everything that has come due. + private func runDueTimers( + delayMs: Double, + context: JSContext, + settlement: SettlementParameters + ) throws { + let start = ContinuousClock.now + let target = start.advanced(by: .milliseconds(Int(delayMs.rounded(.up)))) + + while ContinuousClock.now < target { + try checkInterrupted(settlement) + // Short slices so cancellation and the deadline stay responsive + // during a long backoff. + Thread.sleep(forTimeInterval: 0.005) + } + + // Advance by the time that actually passed, so timers that came due while + // we slept fire together. The call returns the callbacks' uncaught errors, + // which cannot propagate to whoever called `setTimeout` and so surface as + // diagnostics the way an uncaught task error does in a real event loop. + let elapsed = (ContinuousClock.now - start).milliseconds + let errors = Self.stringArray( + from: context, + evaluating: "globalThis.__codemode.advanceTimers(\(elapsed))" + ) + + for message in errors { + settlement.invocationContext.recordDiagnostic( + ToolDiagnostic( + severity: .warning, + code: "TIMER_CALLBACK_ERROR", + message: "A setTimeout callback threw and the error was discarded: \(message)", + category: "execution", + suggestions: ["Handle errors inside timer callbacks; they cannot propagate to the caller of setTimeout."] + ) + ) + } + } + + /// Flags bridge calls that failed while the script still reported success. + /// + /// The runtime installs no unhandled-rejection hook, so a forgotten `await` — + /// the most common LLM JavaScript mistake — discards a `CAPABILITY_DENIED` + /// and the agent is told the run completed. This does not prove the failure + /// was unhandled (a deliberate try/catch looks the same), so it is a warning + /// rather than an error, but it makes the silent case visible. + private func recordSwallowedFailures(invocationContext: BridgeInvocationContext) { + let failures = invocationContext.capabilityFailures() + guard failures.isEmpty == false else { + return + } + + let summary = failures + .map { "\($0.capability) (\($0.code))" } + .joined(separator: ", ") + + invocationContext.recordDiagnostic( + ToolDiagnostic( + severity: .warning, + code: "BRIDGE_FAILURES_NOT_SURFACED", + message: "The script completed successfully, but \(failures.count) bridge call(s) failed and the failure did not reach the result: \(summary).", + category: "execution", + suggestions: [ + "If that was not deliberate, check for a missing await — an un-awaited helper's rejection is silently discarded here.", + "If it was deliberate, ignore this; the result is what the script returned.", + ] + ) + ) + } + private static func noReturnValueDiagnostic(for code: String) -> ToolDiagnostic { let mentionsAsyncIIFE = code.contains("(async") && (code.contains("})()") || code.contains("})();")) let message: String @@ -620,16 +801,14 @@ final class BridgeRuntime: @unchecked Sendable { switch watchdog.termination { case .timedOut: - throw toolError( + throw invocationContext.toolError( code: "SEARCH_TIMEOUT", - message: "Search timed out after \(timeoutMs)ms", - transcript: invocationContext + message: "Search timed out after \(timeoutMs)ms" ) case .cancelled: - throw toolError( + throw invocationContext.toolError( code: "CANCELLED", - message: "Search cancelled", - transcript: invocationContext + message: "Search cancelled" ) case nil: break @@ -642,20 +821,24 @@ final class BridgeRuntime: @unchecked Sendable { let settlement = try waitForSettlement( context: context, - watchdog: watchdog, - cancellationController: cancellationController, - invocationContext: invocationContext, - timeoutCode: "SEARCH_TIMEOUT", - timeoutMessage: "Search timed out after \(timeoutMs)ms", - cancelMessage: "Search cancelled" + settlement: SettlementParameters( + watchdog: watchdog, + cancellationController: cancellationController, + invocationContext: invocationContext, + timeoutCode: "SEARCH_TIMEOUT", + timeoutMessage: "Search timed out after \(timeoutMs)ms", + cancelMessage: "Search cancelled" + ) ) - watchdog.rearm(timeoutMs: max(timeoutMs, 1_000)) + watchdog.rearm(timeoutMs: min(timeoutMs, Self.serializationBudgetMs)) switch settlement { case .fulfilled: return try decodeOutput( from: context, + watchdog: watchdog, + invocationContext: invocationContext, errorCode: "INVALID_SEARCH_RESULT", errorMessagePrefix: "Search result must be JSON-serializable" ) @@ -667,6 +850,8 @@ final class BridgeRuntime: @unchecked Sendable { private func decodeOutput( from context: JSContext, + watchdog: ExecutionWatchdog? = nil, + invocationContext: BridgeInvocationContext? = nil, errorCode: String = "INVALID_RESULT", errorMessagePrefix: String = "Execution result must be JSON-serializable" ) throws -> JSONValue? { @@ -674,17 +859,46 @@ final class BridgeRuntime: @unchecked Sendable { return nil } - guard let serialization = context.evaluateScript( + let limits = config.executionLimits + // Size is measured and the result is replaced *inside* JS: bringing a + // gigabyte of JSON across into a Swift String and then rejecting it has + // already spent the memory the cap exists to protect. The replacement is + // still valid JSON, so the model gets a parseable value plus a + // diagnostic rather than a truncated fragment it cannot read. + let serialization = context.evaluateScript( """ (function(){ try { - return { ok: true, json: JSON.stringify(globalThis.__codemode.result) }; + var json = JSON.stringify(globalThis.__codemode.result); + if (typeof json === 'string' && json.length > \(limits.maxResultCharacters)) { + return { + ok: true, + truncated: true, + size: json.length, + json: JSON.stringify({ + __codeModeTruncated: true, + characterCount: json.length, + limit: \(limits.maxResultCharacters), + preview: json.slice(0, \(limits.truncatedResultPreviewCharacters)) + }) + }; + } + return { ok: true, json: json }; } catch (error) { return { ok: false, message: String(error) }; } })() """ - ) else { + ) + + // Checked before anything else: a watchdog termination during + // serialization is a timeout, not a serialization problem, and reporting + // `INVALID_RESULT: must be JSON-serializable` for a runaway getter + // steered the model to fix the wrong thing. + if let termination = watchdog?.termination { + throw terminationError(termination, invocationContext: invocationContext, during: "result serialization") + } + guard let serialization else { throw CodeModeToolError(code: errorCode, message: errorMessagePrefix) } @@ -693,6 +907,22 @@ final class BridgeRuntime: @unchecked Sendable { throw CodeModeToolError(code: errorCode, message: "\(errorMessagePrefix): \(message)") } + if serialization.forProperty("truncated")?.toBool() == true { + let size = serialization.forProperty("size")?.toNumber()?.intValue ?? limits.maxResultCharacters + invocationContext?.recordDiagnostic( + ToolDiagnostic( + severity: .warning, + code: "RESULT_TRUNCATED", + message: "The result serialized to \(size) characters, over the \(limits.maxResultCharacters)-character limit. It was replaced with a descriptor object carrying a \(limits.truncatedResultPreviewCharacters)-character preview.", + category: "execution", + suggestions: [ + "Return a summary, a count, or a slice instead of the whole collection.", + "Write large payloads to the sandbox with apple.fs.write and return the path.", + ] + ) + ) + } + guard let jsonValue = serialization.forProperty("json"), jsonValue.isUndefined == false, jsonValue.isNull == false, let serialized = jsonValue.toString() else { @@ -706,12 +936,37 @@ final class BridgeRuntime: @unchecked Sendable { return try JSONDecoder.codeModeBridge.decode(JSONValue.self, from: data) } + /// Reports a watchdog termination that happened outside the script body — so + /// far, during result serialization — as the timeout or cancellation it + /// actually is. + private func terminationError( + _ termination: ExecutionWatchdog.Termination, + invocationContext: BridgeInvocationContext?, + during phase: String + ) -> CodeModeToolError { + let (code, message): (String, String) = switch termination { + case .timedOut: + ("EXECUTION_TIMEOUT", "Execution timed out during \(phase). A getter or toJSON on the returned value did not finish within the serialization budget.") + case .cancelled: + ("CANCELLED", "Execution cancelled during \(phase)") + } + + return invocationContext?.toolError(code: code, message: message) + ?? CodeModeToolError(code: code, message: message) + } + private func bridgeFailurePayload( for error: BridgeError, capability: CapabilityID?, - capabilityKey: CodeModeCapabilityKey? + capabilityKey: CodeModeCapabilityKey?, + invocationContext: BridgeInvocationContext? = nil ) -> CodeModeToolError { - let (message, suggestions) = enrichedBridgeFailure(for: error, capability: capability, capabilityKey: capabilityKey) + let (message, suggestions) = enrichedBridgeFailure( + for: error, + capability: capability, + capabilityKey: capabilityKey, + invocationContext: invocationContext + ) return CodeModeToolError( code: error.diagnosticCode, message: message, @@ -833,13 +1088,19 @@ final class BridgeRuntime: @unchecked Sendable { } private func rejectedSuggestions(from context: JSContext) -> [String] { - guard let json = context.evaluateScript("JSON.stringify(globalThis.__codemode.error?.suggestions ?? [])")?.toString(), + Self.stringArray(from: context, evaluating: "globalThis.__codemode.error?.suggestions ?? []") + } + + /// Reads a JavaScript array of strings across the bridge. Returns an empty + /// array if the expression is missing or not string-shaped. + private static func stringArray(from context: JSContext, evaluating expression: String) -> [String] { + guard let json = context.evaluateScript("JSON.stringify(\(expression))")?.toString(), let data = json.data(using: .utf8), - let suggestions = try? JSONDecoder.codeModeBridge.decode([String].self, from: data) + let values = try? JSONDecoder.codeModeBridge.decode([String].self, from: data) else { return [] } - return suggestions + return values } private func event(for error: CodeModeToolError) -> JavaScriptExecutionEvent { @@ -855,28 +1116,6 @@ final class BridgeRuntime: @unchecked Sendable { } } - private func toolError( - code: String, - message: String, - transcript: BridgeInvocationContext, - functionName: String? = nil, - capability: CapabilityID? = nil, - capabilityKey: CodeModeCapabilityKey? = nil, - suggestions: [String] = [] - ) -> CodeModeToolError { - CodeModeToolError( - code: code, - message: message, - functionName: functionName, - capability: capability, - capabilityKey: capabilityKey, - suggestions: suggestions, - diagnostics: transcript.allDiagnostics(), - logs: transcript.allLogs(), - permissionEvents: transcript.allPermissionEvents() - ) - } - private func bridgeSuggestions(for capability: CapabilityID?) -> [String] { bridgeSuggestions(for: capability, capabilityKey: capability?.codeModeKey) } @@ -908,14 +1147,34 @@ final class BridgeRuntime: @unchecked Sendable { private func enrichedBridgeFailure( for error: BridgeError, capability: CapabilityID?, - capabilityKey: CodeModeCapabilityKey? + capabilityKey: CodeModeCapabilityKey?, + invocationContext: BridgeInvocationContext? = nil ) -> (String, [String]) { + // A host-withheld denial reads the same whichever allowlist it came from, + // and is the one denial the model cannot repair — so it short-circuits + // both cases below. + let deniedIdentifier: String? = switch error { + case let .capabilityDenied(capability): capability.rawValue + case let .capabilityKeyDenied(capabilityKey): capabilityKey.rawValue + default: nil + } + if let deniedIdentifier, invocationContext?.isWithheldByHost(deniedIdentifier) == true { + return ( + error.localizedDescription, + [ + "The host app's capability grant does not include \"\(deniedIdentifier)\".", + "This is not repaired by adding it to allowedCapabilities or allowedCapabilityKeys; the host owns this decision.", + "Continue with the capabilities you do have, or report the missing access to the user.", + ] + ) + } + let suggestions: [String] switch error { case let .capabilityDenied(capability): suggestions = [ "Add \"\(capability.rawValue)\" to allowedCapabilities and retry.", - "If your host uses a unified capability-key allowlist, add \"\(capability.rawValue)\" to allowedCapabilityKeys.", + "Built-in capabilities are granted only by allowedCapabilities; listing one in allowedCapabilityKeys has no effect.", ] + bridgeSuggestions(for: capability, capabilityKey: capability.codeModeKey) case let .capabilityKeyDenied(capabilityKey): suggestions = [ @@ -1103,3 +1362,12 @@ final class BridgeRuntime: @unchecked Sendable { return trimmed } } + +private extension Duration { + /// Whole and fractional milliseconds. `components` splits into seconds plus + /// attoseconds, which is otherwise an awkward two-term conversion. + var milliseconds: Double { + let parts = components + return Double(parts.seconds) * 1_000 + Double(parts.attoseconds) / 1e15 + } +} diff --git a/Sources/CodeMode/Runtime/ExecutionLimits.swift b/Sources/CodeMode/Runtime/ExecutionLimits.swift new file mode 100644 index 0000000..bcbb891 --- /dev/null +++ b/Sources/CodeMode/Runtime/ExecutionLimits.swift @@ -0,0 +1,66 @@ +import Foundation + +/// Bounds on how much an execution may accumulate. +/// +/// Nothing about a script's *output* was previously capped: the whole result was +/// stringified, copied into a Swift `String`, and re-decoded into a `JSONValue`; +/// every `console.log` was retained and then copied again into any +/// `CodeModeToolError`; and the events `AsyncStream` used unbounded buffering, so +/// a chatty script that the host was not draining buffered without limit. A +/// single script could jetsam an iOS host. +/// +/// For an LLM consumer, a truncated result carrying a diagnostic is strictly +/// better than an out-of-memory kill, so every limit here degrades rather than +/// fails. +public struct ExecutionLimits: Sendable, Equatable { + /// Maximum size of the serialized result, in JSON characters. A larger + /// result is replaced by a descriptor object carrying a prefix of the JSON. + public var maxResultCharacters: Int + + /// How much of the oversized JSON to include in that descriptor. + public var truncatedResultPreviewCharacters: Int + + /// Maximum retained `console.log` entries. + public var maxLogEntries: Int + + /// Maximum characters retained per log message; longer messages are elided + /// in the middle, which keeps both the prefix and the tail readable. + public var maxLogMessageCharacters: Int + + /// Maximum retained diagnostics. + public var maxDiagnostics: Int + + /// Maximum retained permission events. + public var maxPermissionEvents: Int + + /// How many executions may hold a thread and a live JavaScriptCore context + /// at once. Excess executions queue rather than exhausting the GCD thread + /// pool; the per-execution `timeoutMs` measures the run, not the wait. + public var maxConcurrentExecutions: Int + + /// Depth of the events `AsyncStream` buffer. A host that stops draining + /// drops the oldest events rather than growing without bound. + public var maxBufferedEvents: Int + + public init( + maxResultCharacters: Int = 1_000_000, + truncatedResultPreviewCharacters: Int = 4_000, + maxLogEntries: Int = 1_000, + maxLogMessageCharacters: Int = 8_000, + maxDiagnostics: Int = 200, + maxPermissionEvents: Int = 200, + maxBufferedEvents: Int = 1_000, + maxConcurrentExecutions: Int = 8 + ) { + self.maxResultCharacters = maxResultCharacters + self.truncatedResultPreviewCharacters = truncatedResultPreviewCharacters + self.maxLogEntries = maxLogEntries + self.maxLogMessageCharacters = maxLogMessageCharacters + self.maxDiagnostics = maxDiagnostics + self.maxPermissionEvents = maxPermissionEvents + self.maxBufferedEvents = maxBufferedEvents + self.maxConcurrentExecutions = maxConcurrentExecutions + } + + public static let standard = ExecutionLimits() +} diff --git a/Sources/CodeMode/Runtime/ExecutionSupport.swift b/Sources/CodeMode/Runtime/ExecutionSupport.swift index 1b20c1f..4a0db8b 100644 --- a/Sources/CodeMode/Runtime/ExecutionSupport.swift +++ b/Sources/CodeMode/Runtime/ExecutionSupport.swift @@ -19,38 +19,92 @@ final class ExecutionCancellationController: @unchecked Sendable { final class ExecutionTranscript: @unchecked Sendable { private let lock = NSLock() + private let limits: ExecutionLimits private var logs: [ExecutionLog] = [] private var diagnostics: [ToolDiagnostic] = [] private var permissionEvents: [PermissionEvent] = [] + private var noticedLogOverflow = false private let emitEvent: @Sendable (JavaScriptExecutionEvent) -> Void - init(emitEvent: @escaping @Sendable (JavaScriptExecutionEvent) -> Void = { _ in }) { + init( + limits: ExecutionLimits = .standard, + emitEvent: @escaping @Sendable (JavaScriptExecutionEvent) -> Void = { _ in } + ) { + self.limits = limits self.emitEvent = emitEvent } func record(log: ExecutionLog) { + var entry = log + entry.message = Self.elide(entry.message, to: limits.maxLogMessageCharacters) + + // Keep the first N: the beginning of a runaway log is where the cause is, + // and the tail is the same line a million times. lock.lock() - logs.append(log) + let accepted = logs.count < limits.maxLogEntries + let isFirstOverflow = accepted == false && noticedLogOverflow == false + if accepted { + logs.append(entry) + } else { + noticedLogOverflow = true + } lock.unlock() - emitEvent(.log(log)) + + if accepted { + // Streaming stays live even after the retained transcript is full; + // it is the retained copy — re-copied into every CodeModeToolError — + // that is the memory risk. + emitEvent(.log(entry)) + } else if isFirstOverflow { + record( + diagnostic: ToolDiagnostic( + severity: .warning, + code: "LOG_LIMIT_REACHED", + message: "Kept the first \(limits.maxLogEntries) console.log entries; later entries are dropped.", + category: "execution", + suggestions: ["Log summaries rather than per-item lines, or return the data instead of logging it."] + ) + ) + } } func record(diagnostic: ToolDiagnostic) { lock.lock() - diagnostics.append(diagnostic) + let accepted = diagnostics.count < limits.maxDiagnostics + if accepted { + diagnostics.append(diagnostic) + } lock.unlock() - if diagnostic.severity != .error { + // Error-severity diagnostics used to be recorded but never emitted, so a + // host watching the stream saw every diagnostic *except* the ones that + // mattered most. They are on the result either way; the stream is the + // live view and should not be the incomplete one. + if accepted { emitEvent(.diagnostic(diagnostic)) } } func record(permissionEvent: PermissionEvent) { lock.lock() - permissionEvents.append(permissionEvent) + if permissionEvents.count < limits.maxPermissionEvents { + permissionEvents.append(permissionEvent) + } lock.unlock() } + /// Elides the middle of an over-long message, keeping both ends: a stack + /// trace's origin and its outcome are usually at opposite ends. + private static func elide(_ message: String, to limit: Int) -> String { + guard limit > 0, message.count > limit else { + return message + } + let keep = max(1, (limit - 1) / 2) + let head = message.prefix(keep) + let tail = message.suffix(keep) + return "\(head)… [\(message.count - 2 * keep) characters elided] …\(tail)" + } + func snapshot(output: JSONValue? = nil) -> JavaScriptExecutionResult { lock.lock() defer { lock.unlock() } diff --git a/Sources/CodeMode/Runtime/ExecutionWatchdog.swift b/Sources/CodeMode/Runtime/ExecutionWatchdog.swift index a9b8ea4..dba24e7 100644 --- a/Sources/CodeMode/Runtime/ExecutionWatchdog.swift +++ b/Sources/CodeMode/Runtime/ExecutionWatchdog.swift @@ -40,21 +40,28 @@ final class ExecutionWatchdog: @unchecked Sendable { fileprivate static let checkInterval: TimeInterval = 0.05 private let lock = NSLock() - private var deadlineValue: Date + /// `ContinuousClock`, not `Date`: a wall-clock deadline moves when NTP + /// adjusts the clock, which can push a timeout arbitrarily far out (or fire it + /// immediately) while a script is running. + private var deadlineValue: ContinuousClock.Instant private var terminationValue: Termination? private let cancellationController: ExecutionCancellationController init(timeoutMs: Int, cancellationController: ExecutionCancellationController) { - self.deadlineValue = Date().addingTimeInterval(Double(timeoutMs) / 1_000) + self.deadlineValue = ContinuousClock.now.advanced(by: .milliseconds(timeoutMs)) self.cancellationController = cancellationController } - var deadline: Date { + var deadline: ContinuousClock.Instant { lock.lock() defer { lock.unlock() } return deadlineValue } + var hasPassedDeadline: Bool { + ContinuousClock.now >= deadline + } + var termination: Termination? { lock.lock() defer { lock.unlock() } @@ -67,7 +74,7 @@ final class ExecutionWatchdog: @unchecked Sendable { /// user-defined getters/`toJSON`), while a runaway getter is still terminated. func rearm(timeoutMs: Int) { lock.lock() - deadlineValue = Date().addingTimeInterval(Double(timeoutMs) / 1_000) + deadlineValue = ContinuousClock.now.advanced(by: .milliseconds(timeoutMs)) lock.unlock() } @@ -100,7 +107,7 @@ final class ExecutionWatchdog: @unchecked Sendable { terminationValue = .cancelled return true } - if Date() >= deadlineValue { + if ContinuousClock.now >= deadlineValue { terminationValue = .timedOut return true } diff --git a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift index f10ef84..03b88ba 100644 --- a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift +++ b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift @@ -1,6 +1,83 @@ import Foundation enum RuntimeJavaScript { + /// Shared by the execution and search runtimes. + private static let timerRuntime = """ + // A real timer queue, drained by the host between JavaScript turns. + // + // `setTimeout` used to invoke its callback synchronously and ignore the + // delay, which made three things wrong at once: `clearTimeout` could never + // cancel, an error thrown by the callback propagated to setTimeout's *caller* + // rather than being an uncaught task error, and + // `await new Promise(r => setTimeout(r, 2000))` — the standard backoff — was + // a hot loop that hammered remote APIs through `fetch`. + globalThis.__codemode.timers = { nextID: 1, entries: [] }; + + globalThis.setTimeout = function(fn, delay) { + if (typeof fn !== 'function') { + return 0; + } + const timers = globalThis.__codemode.timers; + const id = timers.nextID++; + timers.entries.push({ + id: id, + fn: fn, + args: Array.prototype.slice.call(arguments, 2), + dueIn: Math.max(0, Number(delay) || 0) + }); + return id; + }; + + globalThis.clearTimeout = function(id) { + const timers = globalThis.__codemode.timers; + timers.entries = timers.entries.filter(function(entry){ return entry.id !== id; }); + }; + + // Milliseconds until the earliest pending timer, or -1 when none are queued. + // -1 means nothing can advance the program: under this runtime's synchronous + // bridge there is no other source of asynchrony. + globalThis.__codemode.nextTimerDelay = function() { + const entries = globalThis.__codemode.timers.entries; + if (entries.length === 0) { + return -1; + } + return entries.reduce(function(min, entry){ return Math.min(min, entry.dueIn); }, Infinity); + }; + + globalThis.__codemode.advanceTimers = function(elapsedMs) { + const timers = globalThis.__codemode.timers; + const due = []; + const remaining = []; + timers.entries.forEach(function(entry){ + entry.dueIn -= elapsedMs; + if (entry.dueIn <= 0) { due.push(entry); } else { remaining.push(entry); } + }); + timers.entries = remaining; + // Due time first, registration order only to break ties — the order a + // real event loop fires in. Sorting by id alone was wrong whenever the + // host's sleep overshot far enough for two timers with *different* + // delays to come due in the same tick: the later one fired first purely + // because it was registered first. `dueIn` is now negative for anything + // overdue, so ascending puts the most overdue first. + due.sort(function(a, b){ + return a.dueIn === b.dueIn ? a.id - b.id : a.dueIn - b.dueIn; + }); + const errors = []; + due.forEach(function(entry){ + try { + entry.fn.apply(null, entry.args); + } catch (error) { + // An uncaught error in a task cannot propagate to whoever called + // setTimeout; the host reports it as a diagnostic instead. + errors.push(String(error)); + } + }); + // Returned rather than buffered, so the host needs one round trip per + // tick instead of two. + return errors; + }; + """ + static func pruningScript(removingJavaScriptNames names: some Sequence) -> String { let bindingsToRemove = Array(Set(names)).sorted() @@ -123,13 +200,7 @@ enum RuntimeJavaScript { error: function(){ __searchConsole('error', Array.from(arguments).map(function(v){ return String(v); }).join(' ')); } }; - globalThis.setTimeout = function(fn, delay) { - if (typeof fn === 'function') { - fn(); - } - return 0; - }; - globalThis.clearTimeout = function(_) {}; + \(timerRuntime) """ static let bootstrap = """ @@ -200,13 +271,7 @@ enum RuntimeJavaScript { error: function(){ __nativeConsoleLog(Array.from(arguments).map(function(v){ return String(v); }).join(' ')); } }; - globalThis.setTimeout = function(fn, delay) { - if (typeof fn === 'function') { - fn(); - } - return 0; - }; - globalThis.clearTimeout = function(_) {}; + \(timerRuntime) if (typeof URLSearchParams === 'undefined') { globalThis.URLSearchParams = function(initial){ @@ -258,16 +323,32 @@ enum RuntimeJavaScript { }; globalThis.apple = globalThis.apple || {}; + // These accept either the object form the catalog advertises — + // apple.keychain.get({ key }) — or the positional form the examples have + // always shown. The catalog described object arguments while the wrapper took + // a positional string, so code written from the metadata sent "[object + // Object]" as the key. + function __keychainArgs(first, value) { + if (first && typeof first === 'object') { + const args = { key: String(first.key) }; + if (first.value !== undefined) args.value = String(first.value); + return args; + } + const args = { key: String(first) }; + if (value !== undefined) args.value = String(value); + return args; + } + globalThis.apple.keychain = { - get: function(key) { return __invokeAsync('keychain.read', { key: String(key) }); }, - set: function(key, value) { return __invokeAsync('keychain.write', { key: String(key), value: String(value) }); }, - delete: function(key) { return __invokeAsync('keychain.delete', { key: String(key) }); } + get: function(key) { return __invokeAsync('keychain.read', __keychainArgs(key)); }, + set: function(key, value) { return __invokeAsync('keychain.write', __keychainArgs(key, value)); }, + delete: function(key) { return __invokeAsync('keychain.delete', __keychainArgs(key)); } }; globalThis.apple.location = { - getPermissionStatus: function() { return __invokeAsync('location.read', { mode: 'permissionStatus' }); }, + getPermissionStatus: function(args) { return __invokeAsync('location.read', Object.assign({ mode: 'permissionStatus' }, args || {})); }, requestPermission: function() { return __invokeAsync('location.permission.request', {}); }, - getCurrentPosition: function() { return __invokeAsync('location.read', { mode: 'current' }); } + getCurrentPosition: function(args) { return __invokeAsync('location.read', Object.assign({ mode: 'current' }, args || {})); } }; globalThis.apple.calendar = { @@ -353,6 +434,17 @@ enum RuntimeJavaScript { open: function(args) { return __invokeAsync('settings.ui.open', args || {}); } }; + // fs.move / fs.copy refuse an existing destination without an explicit + // overwrite, so the Node-style aliases need a way to pass one through. + function __fsDestinationArgs(from, to, options) { + const args = { from: String(from), to: String(to) }; + if (options && typeof options === 'object') { + if (options.overwrite !== undefined) args.overwrite = !!options.overwrite; + if (options.recursive !== undefined) args.recursive = !!options.recursive; + } + return args; + } + globalThis.fs = { promises: { readFile: function(path, options) { @@ -378,8 +470,12 @@ enum RuntimeJavaScript { rm: function(path, options) { return __invokeAsync('fs.delete', { path: String(path), recursive: !!(options && options.recursive) }); }, - rename: function(from, to) { return __invokeAsync('fs.move', { from: String(from), to: String(to) }); }, - copyFile: function(from, to) { return __invokeAsync('fs.copy', { from: String(from), to: String(to) }); } + rename: function(from, to, options) { + return __invokeAsync('fs.move', __fsDestinationArgs(from, to, options)); + }, + copyFile: function(from, to, options) { + return __invokeAsync('fs.copy', __fsDestinationArgs(from, to, options)); + } } }; diff --git a/Sources/CodeMode/Security/CapabilityGrant.swift b/Sources/CodeMode/Security/CapabilityGrant.swift new file mode 100644 index 0000000..af300cd --- /dev/null +++ b/Sources/CodeMode/Security/CapabilityGrant.swift @@ -0,0 +1,75 @@ +import Foundation + +/// The host-owned ceiling on what an execution may reach. +/// +/// `JavaScriptExecutionRequest.allowedCapabilities` / `allowedCapabilityKeys` are +/// *model-authored* — they travel in the advertised tool schema, so a script +/// declares them for itself. They are a least-privilege declaration of intent and +/// a useful audit signal, but they are not a sandbox: on their own, a script that +/// asks for `keychain.read` gets it. +/// +/// `CapabilityGrant` is the half the host owns. It is configured on +/// `CodeModeConfiguration` and intersected with the request before the invocation +/// context is built, so the effective set is always +/// `requested ∩ granted`. Anything the model asks for that the grant withholds +/// fails with `CAPABILITY_DENIED` and is reported to the model as *not* +/// repairable by retrying with a wider `allowedCapabilities`. +public struct CapabilityGrant: Sendable, Equatable { + /// Built-in capabilities the host permits, or `nil` for "no host ceiling". + public var capabilities: Set? + /// Custom provider capability keys the host permits, or `nil` for "no host ceiling". + public var capabilityKeys: Set? + + public init(capabilities: Set?, capabilityKeys: Set?) { + self.capabilities = capabilities + self.capabilityKeys = capabilityKeys + } + + /// No host ceiling: whatever the request declares is what the script gets. + /// + /// This is the default for source compatibility, and it is the right choice + /// only when the host has already vetted the request itself. Hosts that pipe + /// model-authored tool JSON straight into `executeJavaScript` should set an + /// explicit grant. + public static let unrestricted = CapabilityGrant(capabilities: nil, capabilityKeys: nil) + + /// Grants nothing; every capability request is withheld. + public static let none = CapabilityGrant(capabilities: [], capabilityKeys: []) + + /// Grants exactly the listed built-in capabilities and provider keys. + public static func only( + _ capabilities: Set, + capabilityKeys: Set = [] + ) -> CapabilityGrant { + CapabilityGrant(capabilities: capabilities, capabilityKeys: capabilityKeys) + } + + /// Applies the ceiling to a model-authored request. + /// + /// Returns the effective sets alongside everything that was asked for and + /// withheld, so the caller can tell the model why a retry will not help. + public func resolve( + requestedCapabilities: Set, + requestedCapabilityKeys: Set + ) -> Resolution { + let effectiveCapabilities = capabilities.map { requestedCapabilities.intersection($0) } ?? requestedCapabilities + let effectiveKeys = capabilityKeys.map { requestedCapabilityKeys.intersection($0) } ?? requestedCapabilityKeys + + return Resolution( + capabilities: effectiveCapabilities, + capabilityKeys: effectiveKeys, + withheld: ( + requestedCapabilities.subtracting(effectiveCapabilities).map(\.rawValue) + + requestedCapabilityKeys.subtracting(effectiveKeys).map(\.rawValue) + ).sorted() + ) + } + + public struct Resolution: Sendable, Equatable { + public var capabilities: Set + public var capabilityKeys: Set + /// Identifiers the request declared and the grant refused, sorted. Both + /// namespaces together, because every consumer reports them as one list. + public var withheld: [String] + } +} diff --git a/Sources/CodeMode/Security/NetworkAccessPolicy.swift b/Sources/CodeMode/Security/NetworkAccessPolicy.swift index 3dd10f8..6bb8e80 100644 --- a/Sources/CodeMode/Security/NetworkAccessPolicy.swift +++ b/Sources/CodeMode/Security/NetworkAccessPolicy.swift @@ -5,8 +5,11 @@ import Foundation /// /// The default (`.standard`) blocks loopback, private-range, link-local, and /// other non-public destinations so agent-authored JavaScript cannot reach -/// `127.0.0.1`, cloud metadata endpoints such as `169.254.169.254`, or hosts on -/// the local network, and caps buffered response bodies at 10 MB. +/// `127.0.0.1`, cloud metadata endpoints such as `169.254.169.254`, or names that +/// resolve onto the local network, and caps buffered response bodies at 10 MB. +/// "Names that resolve onto the local network" covers `localhost`, +/// `*.local`/`*.localhost`/`*.internal`, and bare single-label hosts such as +/// `router` or `intranet`, which reach the LAN through the DHCP search domain. /// /// Host matching is by URL host only; DNS resolution is not performed, so a /// public hostname that resolves to a private address is not detected. Hosts @@ -31,28 +34,64 @@ public struct NetworkAccessPolicy: Sendable, Equatable { /// that exceed this are cancelled mid-transfer. public var maxResponseBytes: Int + /// Whether script-supplied credential headers reach the wire. + /// + /// `Cookie`, `Authorization`, and the `Proxy-*` pair are how a request + /// carries authority. Script-authored code should not be minting them + /// against arbitrary origins, so they are refused by default. Hosts whose + /// scripts legitimately call an authenticated API — a bearer token the + /// script fetched from the keychain, say — set this to true. + public var allowsCredentialHeaders: Bool + + /// Request headers refused when `allowsCredentialHeaders` is false. + /// Matched case-insensitively; `proxy-` is matched as a prefix. + static let credentialHeaderNames: Set = [ + "cookie", + "cookie2", + "authorization", + ] + public init( allowedHosts: [String]? = nil, blockedHosts: [String] = [], blocksPrivateNetworks: Bool = true, - maxResponseBytes: Int = 10_485_760 + maxResponseBytes: Int = 10_485_760, + allowsCredentialHeaders: Bool = false ) { self.allowedHosts = allowedHosts self.blockedHosts = blockedHosts self.blocksPrivateNetworks = blocksPrivateNetworks self.maxResponseBytes = maxResponseBytes + self.allowsCredentialHeaders = allowsCredentialHeaders } - /// Secure default: private networks blocked, 10 MB response cap. + /// Secure default: private networks blocked, 10 MB response cap, no + /// script-supplied credential headers. public static let standard = NetworkAccessPolicy() - /// No destination restrictions and no response size cap. Matches the - /// pre-policy behavior of `network.fetch`; opt in deliberately. + /// No destination restrictions, no response size cap, and script-supplied + /// credential headers permitted. Matches the pre-policy behavior of + /// `network.fetch`; opt in deliberately. public static let permissive = NetworkAccessPolicy( blocksPrivateNetworks: false, - maxResponseBytes: .max + maxResponseBytes: .max, + allowsCredentialHeaders: true ) + /// Returns a reason the header is refused, or nil when it may be sent. + func headerViolationReason(for name: String) -> String? { + guard allowsCredentialHeaders == false else { + return nil + } + + let normalized = name.lowercased() + guard Self.credentialHeaderNames.contains(normalized) || normalized.hasPrefix("proxy-") else { + return nil + } + + return "the \"\(name)\" request header carries authority and is refused by the host app's network access policy" + } + /// Returns a human-readable reason the URL is refused, or nil when allowed. func violationReason(for url: URL) -> String? { guard let rawHost = url.host, rawHost.isEmpty == false else { @@ -119,6 +158,14 @@ public struct NetworkAccessPolicy: Sendable, Equatable { return isPrivateOrLocal(ipv6: ipv6) } + // A bare single-label name — `router`, `intranet`, `nas`, `wpad` — is not + // a public destination. It resolves through the DHCP-supplied search + // domain, so it lands on the local network exactly like `*.local` does, + // and it previously fell straight through to "allowed". + if host.contains(".") == false { + return true + } + return false } diff --git a/Sources/CodeMode/Security/PathPolicy.swift b/Sources/CodeMode/Security/PathPolicy.swift index 6ba31f8..45746b0 100644 --- a/Sources/CodeMode/Security/PathPolicy.swift +++ b/Sources/CodeMode/Security/PathPolicy.swift @@ -23,6 +23,26 @@ public struct PathPolicyConfig: Sendable { public protocol PathPolicy: Sendable { func resolve(path: String) throws -> URL + + /// The roots this policy admits. + /// + /// `resolve` accepts a root itself — `documents:` resolves to the documents + /// root — so destructive operations need to be able to recognize one and + /// refuse. Custom policies that do not implement this get an empty list and + /// simply lose that specific check. + var allowedRoots: [URL] { get } +} + +public extension PathPolicy { + var allowedRoots: [URL] { [] } + + /// True when `url` *is* one of the allowed roots rather than something inside one. + func isAllowedRoot(_ url: URL) -> Bool { + let candidate = url.standardizedFileURL.resolvingSymlinksInPath().standardizedFileURL.path + return allowedRoots.contains { root in + root.standardizedFileURL.resolvingSymlinksInPath().standardizedFileURL.path == candidate + } + } } public struct DefaultPathPolicy: PathPolicy { @@ -32,6 +52,10 @@ public struct DefaultPathPolicy: PathPolicy { self.config = config } + public var allowedRoots: [URL] { + [config.tmpRoot, config.cachesRoot, config.documentsRoot, config.appGroupRoot].compactMap { $0 } + } + public func resolve(path: String) throws -> URL { let cleaned = path.trimmingCharacters(in: .whitespacesAndNewlines) guard cleaned.isEmpty == false else { @@ -98,15 +122,54 @@ public struct DefaultPathPolicy: PathPolicy { var missingComponents: [String] = [] let fileManager = FileManager.default - while fileManager.fileExists(atPath: existingAncestor.path) == false, + // `fileExists` follows symlinks, so a symlink pointing at a *nonexistent* + // out-of-root target reported "missing" and was treated as a component to + // re-append rather than a link to resolve — admitting it. Checking for the + // link itself keeps a dangling symlink in the resolution path, where + // `resolvingSymlinksInPath` sends containment to the real target. + while Self.isMissingComponent(existingAncestor, fileManager: fileManager), existingAncestor.path != existingAncestor.deletingLastPathComponent().path { missingComponents.insert(existingAncestor.lastPathComponent, at: 0) existingAncestor.deleteLastPathComponent() } - let resolvedAncestor = existingAncestor.resolvingSymlinksInPath().standardizedFileURL + let resolvedAncestor = Self.followingDanglingSymlink( + existingAncestor.resolvingSymlinksInPath().standardizedFileURL, + fileManager: fileManager + ) return missingComponents.reduce(resolvedAncestor) { partial, component in partial.appendingPathComponent(component) }.standardizedFileURL } + + /// True when nothing exists at this path — not even a broken symlink. + /// `FileManager.fileExists` follows links and so answers false for one. + private static func isMissingComponent(_ url: URL, fileManager: FileManager) -> Bool { + if fileManager.fileExists(atPath: url.path) { + return false + } + // `attributesOfItem` does not traverse the final symlink, so a dangling + // link still has attributes here. + return (try? fileManager.attributesOfItem(atPath: url.path)) == nil + } + + /// Resolves a symlink whose target does not exist. + /// + /// `URL.resolvingSymlinksInPath()` only rewrites links it can follow, so a + /// link to a nonexistent out-of-root target came back unchanged and passed + /// containment. Containment must be judged on where the link *points*. + private static func followingDanglingSymlink(_ url: URL, fileManager: FileManager) -> URL { + var current = url + // Bounded: a symlink cycle would otherwise spin here. + for _ in 0..<16 { + guard let destination = try? fileManager.destinationOfSymbolicLink(atPath: current.path) else { + return current + } + let next = destination.hasPrefix("/") + ? URL(fileURLWithPath: destination) + : current.deletingLastPathComponent().appendingPathComponent(destination) + current = next.standardizedFileURL.resolvingSymlinksInPath().standardizedFileURL + } + return current + } } diff --git a/Sources/CodeMode/Security/SystemPermissionBroker.swift b/Sources/CodeMode/Security/SystemPermissionBroker.swift index 27f4451..27fd3f1 100644 --- a/Sources/CodeMode/Security/SystemPermissionBroker.swift +++ b/Sources/CodeMode/Security/SystemPermissionBroker.swift @@ -45,6 +45,18 @@ import HomeKit #endif public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable { + /// How long a `request(for:)` waits on the system's TCC dialog before giving + /// up and re-reading the status. This is a human-latency budget, and it is + /// short: a user who takes longer than this leaves the caller reading the + /// pre-prompt status. + static let permissionPromptTimeoutSeconds: TimeInterval = 10 + + /// Every TCC request in this type waits that same budget, so the timeout is + /// not repeated at six call sites where it could drift apart. + private static func awaitingPrompt(_ request: (@escaping @Sendable () -> Void) -> Void) { + CompletionWait.completion(timeout: permissionPromptTimeoutSeconds, request) + } + public init() {} public func status(for permission: PermissionKind) -> PermissionStatus { @@ -111,8 +123,15 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func locationStatus() -> PermissionStatus { #if canImport(CoreLocation) - let manager = CLLocationManager() - switch manager.authorizationStatus { + return Self.mapLocationAuthorization(CLLocationManager().authorizationStatus) + #else + return .unavailable + #endif + } + + #if canImport(CoreLocation) + static func mapLocationAuthorization(_ status: CLAuthorizationStatus) -> PermissionStatus { + switch status { case .authorizedWhenInUse, .authorizedAlways: return .granted case .denied: @@ -124,10 +143,8 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable @unknown default: return .unavailable } - #else - return .unavailable - #endif } + #endif private func contactsStatus() -> PermissionStatus { #if canImport(Contacts) @@ -320,9 +337,47 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable #endif } + /// Reads HomeKit authorization *without* constructing a manager. + /// + /// Constructing `HMHomeManager` is itself what triggers the HomeKit TCC + /// prompt, so the previous implementation put a user-facing dialog inside a + /// status check — and `requestHomeKitPermission` called it twice, then built a + /// third manager. A status query now answers from the shared manager if one + /// already exists and otherwise fails closed with `.notDetermined`, which + /// routes the caller through the request path where a prompt belongs. private func homeKitStatus() -> PermissionStatus { #if canImport(HomeKit) - let status = HMHomeManager().authorizationStatus + guard let manager = Self.existingHomeManager() else { + return .notDetermined + } + return Self.mapHomeKitAuthorization(manager.authorizationStatus) + #else + return .unavailable + #endif + } + + #if canImport(HomeKit) + /// Process-wide so repeated permission work reuses one manager instead of + /// re-triggering the prompt. + private static let homeManagerBox = LockedBox(nil) + + private static func existingHomeManager() -> HMHomeManager? { + homeManagerBox.get() + } + + /// Creates the shared manager if needed. This is the call that prompts. + private static func makeHomeManager(delegate: HMHomeManagerDelegate) -> HMHomeManager { + if let existing = homeManagerBox.get() { + existing.delegate = delegate + return existing + } + let manager = HMHomeManager() + manager.delegate = delegate + homeManagerBox.set(manager) + return manager + } + + private static func mapHomeKitAuthorization(_ status: HMHomeManagerAuthorizationStatus) -> PermissionStatus { if status.contains(.authorized) { return .granted } @@ -333,10 +388,8 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable return .notDetermined } return .unavailable - #else - return .unavailable - #endif } + #endif private func speechRecognitionStatus() -> PermissionStatus { #if canImport(Speech) @@ -419,24 +472,41 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestLocationPermission() -> PermissionStatus { #if canImport(CoreLocation) && os(iOS) - let manager = CLLocationManager() let delegate = LocationPermissionDelegate() - manager.delegate = delegate if Thread.isMainThread { + let manager = CLLocationManager() + manager.delegate = delegate + delegate.hold(manager) manager.requestWhenInUseAuthorization() + // Cannot block main for the callback; the caller sees the pre-prompt + // status and a later call observes the resolved one. return locationStatus() } - // async, not sync: execution runs on a background queue, and the host may be - // blocking the main thread waiting on the result — main.sync would deadlock. - // If main never runs the request, the delegate wait below times out instead. + // `CLLocationManager` delivers delegate callbacks on the run loop of the + // thread that *created* it, and the bridge's GCD workers have none — so a + // manager constructed here never called back, the wait below always burned + // its full 10s, and the result was reported from a status re-read that + // races the TCC write. Creating the manager, assigning the delegate, and + // requesting all happen on main. + // + // async, not sync: the host may be blocking main waiting on our result, so + // main.sync would deadlock. If main never runs this, the wait times out. DispatchQueue.main.async { + let manager = CLLocationManager() + manager.delegate = delegate + delegate.hold(manager) manager.requestWhenInUseAuthorization() } - _ = delegate.wait(timeout: 10) - return locationStatus() + guard delegate.wait(timeout: 10) == .success else { + return locationStatus() + } + + // Prefer the status the delegate callback carried over a fresh read, + // which can still observe the pre-prompt value. + return delegate.observedStatus().map(Self.mapLocationAuthorization) ?? locationStatus() #else return .unavailable #endif @@ -444,12 +514,10 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestContactsPermission() -> PermissionStatus { #if canImport(Contacts) - let semaphore = DispatchSemaphore(value: 0) let store = CNContactStore() - store.requestAccess(for: .contacts) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + store.requestAccess(for: .contacts) { _, _ in complete() } } - _ = semaphore.wait(timeout: .now() + 10) return contactsStatus() #else return .unavailable @@ -459,19 +527,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestCalendarPermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 17.0, macOS 14.0, *) { - store.requestFullAccessToEvents { _, _ in - semaphore.signal() - } - } else { - store.requestAccess(to: .event) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + if #available(iOS 17.0, macOS 14.0, *) { + store.requestFullAccessToEvents { _, _ in complete() } + } else { + store.requestAccess(to: .event) { _, _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return calendarReadStatus() #else return .unavailable @@ -481,19 +543,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestCalendarWritePermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 17.0, macOS 14.0, *) { - store.requestWriteOnlyAccessToEvents { _, _ in - semaphore.signal() - } - } else { - store.requestAccess(to: .event) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + if #available(iOS 17.0, macOS 14.0, *) { + store.requestWriteOnlyAccessToEvents { _, _ in complete() } + } else { + store.requestAccess(to: .event) { _, _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return calendarWriteStatus() #else return .unavailable @@ -503,19 +559,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestRemindersPermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 17.0, macOS 14.0, *) { - store.requestFullAccessToReminders { _, _ in - semaphore.signal() - } - } else { - store.requestAccess(to: .reminder) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + if #available(iOS 17.0, macOS 14.0, *) { + store.requestFullAccessToReminders { _, _ in complete() } + } else { + store.requestAccess(to: .reminder) { _, _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return remindersStatus() #else return .unavailable @@ -524,19 +574,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestPhotoLibraryPermission() -> PermissionStatus { #if canImport(Photos) - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 14.0, macOS 11.0, *) { - PHPhotoLibrary.requestAuthorization(for: .readWrite) { _ in - semaphore.signal() - } - } else { - PHPhotoLibrary.requestAuthorization { _ in - semaphore.signal() + Self.awaitingPrompt { complete in + if #available(iOS 14.0, macOS 11.0, *) { + PHPhotoLibrary.requestAuthorization(for: .readWrite) { _ in complete() } + } else { + PHPhotoLibrary.requestAuthorization { _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return photoLibraryStatus() #else return .unavailable @@ -545,13 +589,11 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestNotificationsPermission() -> PermissionStatus { #if canImport(UserNotifications) - let semaphore = DispatchSemaphore(value: 0) - - UNUserNotificationCenter.current().requestAuthorization(options: [.alert, .badge, .sound]) { _, _ in - semaphore.signal() + Self.awaitingPrompt { complete in + UNUserNotificationCenter.current().requestAuthorization(options: [.alert, .badge, .sound]) { _, _ in + complete() + } } - - _ = semaphore.wait(timeout: .now() + 10) return notificationsStatus() #else return .unavailable @@ -560,15 +602,17 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestHomeKitPermission() -> PermissionStatus { #if canImport(HomeKit) - if homeKitStatus() != .notDetermined { - return homeKitStatus() + // One status read, not two, and one manager, not three: each construction + // is a potential prompt. + let current = homeKitStatus() + if current != .notDetermined, Self.existingHomeManager() != nil { + return current } let delegate = HomeKitPermissionDelegate() - let manager = HMHomeManager() - manager.delegate = delegate - _ = delegate.wait(timeout: 10) - return homeKitStatus() + let manager = Self.makeHomeManager(delegate: delegate) + _ = delegate.wait(timeout: Self.permissionPromptTimeoutSeconds) + return Self.mapHomeKitAuthorization(manager.authorizationStatus) #else return .unavailable #endif @@ -667,13 +711,40 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable } #if canImport(CoreLocation) && os(iOS) -private final class LocationPermissionDelegate: NSObject, CLLocationManagerDelegate { +/// `@unchecked Sendable` because it is genuinely thread-safe and has to cross an +/// isolation boundary: the manager is created inside a `DispatchQueue.main.async` +/// block — main-actor-isolated on iOS — while `wait`/`observedStatus` are called +/// from the requesting thread. All mutable state is lock-guarded and the handoff +/// is a semaphore. +private final class LocationPermissionDelegate: NSObject, CLLocationManagerDelegate, @unchecked Sendable { private let semaphore = DispatchSemaphore(value: 0) + private let lock = NSLock() + private var status: CLAuthorizationStatus? + /// The manager is created inside a `DispatchQueue.main.async` block and would + /// otherwise be released as soon as that block returns — before the callback + /// it exists to receive. `CLLocationManager` does not retain its delegate, so + /// the ownership runs the other way here. + private var manager: CLLocationManager? + + func hold(_ manager: CLLocationManager) { + lock.lock() + self.manager = manager + lock.unlock() + } func locationManagerDidChangeAuthorization(_ manager: CLLocationManager) { + lock.lock() + status = manager.authorizationStatus + lock.unlock() semaphore.signal() } + func observedStatus() -> CLAuthorizationStatus? { + lock.lock() + defer { lock.unlock() } + return status + } + func wait(timeout: TimeInterval) -> DispatchTimeoutResult { semaphore.wait(timeout: .now() + timeout) } diff --git a/Sources/CodeMode/Support/CodeModeDate.swift b/Sources/CodeMode/Support/CodeModeDate.swift new file mode 100644 index 0000000..e3c89d7 --- /dev/null +++ b/Sources/CodeMode/Support/CodeModeDate.swift @@ -0,0 +1,58 @@ +import Foundation + +/// The single ISO 8601 parser for every bridge. +/// +/// Two spellings were previously in circulation: Health and Alarm tried +/// `.withFractionalSeconds` as a fallback, while EventKit, Notifications, and the +/// system-UI validators used a bare `ISO8601DateFormatter`. So +/// `2026-07-24T10:00:00.000Z` — a spelling models emit constantly, and what +/// `Date.toISOString()` produces in JavaScript — parsed in some bridges and not +/// others. +enum CodeModeDate { + /// Parses an ISO 8601 timestamp with or without fractional seconds. + static func parse(_ text: String?) -> Date? { + guard let text else { + return nil + } + + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard trimmed.isEmpty == false else { + return nil + } + + let formatter = ISO8601DateFormatter() + if let date = formatter.date(from: trimmed) { + return date + } + + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return formatter.date(from: trimmed) + } + + /// Parses a timestamp that the caller requires, throwing a repairable error + /// rather than substituting a default. + /// + /// Silently falling back to "now" (or "now + 14 days") gave the script a + /// plausible but wrong window with no diagnostic — the worst failure mode + /// available, because the model has no way to notice it happened. + static func require(_ text: String?, argument: String, capability: String) throws -> Date { + guard let text, text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false else { + throw BridgeError.invalidArguments("\(capability) requires '\(argument)'") + } + guard let date = parse(text) else { + throw BridgeError.invalidArguments( + "\(capability) could not parse '\(argument)' as an ISO 8601 timestamp: \"\(text)\". Use a form like 2026-07-24T10:00:00Z." + ) + } + return date + } + + /// Parses an optional timestamp, throwing when a value is present but + /// unparseable. A missing value returns nil and the caller's default applies. + static func optional(_ text: String?, argument: String, capability: String) throws -> Date? { + guard let text, text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false else { + return nil + } + return try require(text, argument: argument, capability: capability) + } +} diff --git a/Sources/CodeMode/Support/CompletionWait.swift b/Sources/CodeMode/Support/CompletionWait.swift new file mode 100644 index 0000000..38f54f4 --- /dev/null +++ b/Sources/CodeMode/Support/CompletionWait.swift @@ -0,0 +1,55 @@ +import Foundation + +/// Bridges a callback-style system API into the bridges' synchronous world. +/// +/// The pattern this replaces — a `var` written from the framework's completion +/// queue, read after `_ = semaphore.wait(...)` with the timeout result +/// discarded — is a data race on timeout: the late callback writes while the +/// caller reads, and the caller cannot tell "timed out" from "no results". Every +/// wait here either produces the value the callback delivered or throws +/// `BridgeError.timeout`; the box stays alive for a late callback to write into +/// harmlessly. +enum CompletionWait { + /// Runs `operation`, which must call its `complete` argument exactly once, + /// and returns the delivered value. + /// + /// - Throws: `BridgeError.timeout` if `complete` is not called in time. + static func value( + timeout: TimeInterval, + operationName: String, + _ operation: (@escaping @Sendable (Value) -> Void) -> Void + ) throws -> Value { + let box = LockedBox(nil) + let semaphore = DispatchSemaphore(value: 0) + + operation { value in + box.set(value) + semaphore.signal() + } + + guard semaphore.wait(timeout: .now() + timeout) == .success else { + throw BridgeError.timeout(milliseconds: Int(timeout * 1_000)) + } + + guard let value = box.get() else { + throw BridgeError.nativeFailure("\(operationName) completed without a result") + } + + return value + } + + /// Waits for a completion that carries no value, for APIs whose result is + /// read back from the store afterwards. + /// + /// - Returns: true when the callback arrived, false on timeout. Callers that + /// must distinguish the two should use `value(timeout:operationName:_:)`. + @discardableResult + static func completion( + timeout: TimeInterval, + _ operation: (@escaping @Sendable () -> Void) -> Void + ) -> Bool { + let semaphore = DispatchSemaphore(value: 0) + operation { semaphore.signal() } + return semaphore.wait(timeout: .now() + timeout) == .success + } +} diff --git a/Sources/CodeMode/Support/JSONValue.swift b/Sources/CodeMode/Support/JSONValue.swift index d96eec3..0442e13 100644 --- a/Sources/CodeMode/Support/JSONValue.swift +++ b/Sources/CodeMode/Support/JSONValue.swift @@ -131,13 +131,34 @@ public extension JSONValue { return nil } + /// Integer view of a JSON number, or `nil` when the value cannot be + /// represented. + /// + /// `Int.init(_: Double)` *traps* on NaN, infinity, and anything outside + /// `Int`'s range, and every value here came from adversarial JSON — a model + /// tool call or a script's `fetch(url, { timeoutMs: 1e300 })`. The conversion + /// has to be total or a single number kills the host process. var intValue: Int? { if case let .number(value) = self { - return Int(value) + return JSONValue.exactInt(from: value) } return nil } + /// Total `Double` → `Int` conversion. Rejects non-finite values and anything + /// the conversion cannot represent; truncates toward zero otherwise, matching + /// the previous behavior for in-range input. + static func exactInt(from value: Double) -> Int? { + guard value.isFinite else { return nil } + let truncated = value.rounded(.towardZero) + guard truncated >= -9_223_372_036_854_775_808.0, + truncated < 9_223_372_036_854_775_808.0 + else { + return nil + } + return Int(truncated) + } + var doubleValue: Double? { if case let .number(value) = self { return value diff --git a/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift b/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift index c3e95b3..f4d8633 100644 --- a/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift +++ b/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift @@ -96,7 +96,6 @@ extension UIKitSystemUIPresenter { public func presentNewCalendarEvent(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let timeoutMs = arguments.int("timeoutMs") ?? Self.defaultTimeoutMs - let formatter = ISO8601DateFormatter() let token = UUID() return try runUIOperation(timeoutMs: timeoutMs, onTimeout: { self.releaseCoordinator(token) }) { complete in @@ -108,9 +107,9 @@ extension UIKitSystemUIPresenter { event.notes = arguments.string("notes") event.location = arguments.string("location") - let startDate = arguments.string("start").flatMap { formatter.date(from: $0) } ?? Date() + let startDate = CodeModeDate.parse(arguments.string("start")) ?? Date() event.startDate = startDate - event.endDate = arguments.string("end").flatMap { formatter.date(from: $0) } + event.endDate = CodeModeDate.parse(arguments.string("end")) ?? Calendar.current.date(byAdding: .hour, value: 1, to: startDate) ?? startDate event.calendar = store.defaultCalendarForNewEvents diff --git a/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift b/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift index 8fd4a7c..e3a5f8b 100644 --- a/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift +++ b/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift @@ -365,7 +365,12 @@ extension UIKitSystemUIPresenter { public func presentWeb(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let timeoutMs = arguments.int("timeoutMs") ?? Self.defaultTimeoutMs - let url = URL(string: arguments.string("url") ?? "")! + // The bridge parses the URL first, but UIKitSystemUIPresenter is public + // API and a host may call it directly — a force unwrap here crashes the + // app on a malformed string. + guard let url = URL(string: arguments.string("url") ?? "") else { + throw BridgeError.invalidArguments("web.ui.present requires an absolute 'url'") + } let token = UUID() #if os(iOS) @@ -402,7 +407,9 @@ extension UIKitSystemUIPresenter { public func authenticateWeb(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let timeoutMs = arguments.int("timeoutMs") ?? Self.defaultTimeoutMs - let url = URL(string: arguments.string("url") ?? "")! + guard let url = URL(string: arguments.string("url") ?? "") else { + throw BridgeError.invalidArguments("auth.ui.webAuthenticate requires an absolute 'url'") + } let callbackURLScheme = arguments.string("callbackURLScheme") let token = UUID() @@ -446,7 +453,11 @@ extension UIKitSystemUIPresenter { } } session.presentationContextProvider = coordinator - session.prefersEphemeralWebBrowserSession = arguments.bool("prefersEphemeralSession") ?? false + // Ephemeral by default: a script-initiated OAuth flow inside the + // user's live Safari session starts out already signed in as the + // user and hands the callback URL — with its code or token — to + // the script. Opting out is a deliberate host/script choice. + session.prefersEphemeralWebBrowserSession = arguments.bool("prefersEphemeralSession") ?? true coordinator.session = session self.retainCoordinator(coordinator, token: token) diff --git a/Sources/CodeModeAuthoring/README.md b/Sources/CodeModeAuthoring/README.md index 57069fa..7a992e9 100644 --- a/Sources/CodeModeAuthoring/README.md +++ b/Sources/CodeModeAuthoring/README.md @@ -69,6 +69,33 @@ let call = try await tools.executeJavaScript( ) ``` +The two allowlists are strictly disjoint: a built-in capability ID listed in +`allowedCapabilityKeys` is ignored, and a provider key listed in +`allowedCapabilities` will not decode. Both fields are model-authored, so treat +them as a declaration of intent — set `CodeModeConfiguration.capabilityGrant` for +the host-owned ceiling that actually enforces access. + Macro v1 maps one type to one JavaScript function. `Arguments` must be a nested struct, and no-arg tools use an empty `Arguments` struct. `call(arguments:)` must be `throws` or `async throws`, and it can return `Void` or a nested `Result` struct. -Supported `Arguments` and `Result` property shapes are JSON primitives, `JSONValue`, arrays/dictionaries of JSON-shaped values, and optional forms. Throw `CodeModeFunctionError` for structured failures that should surface as CodeMode bridge errors. +Supported `Arguments` and `Result` property shapes are JSON primitives, `JSONValue`, arrays/dictionaries of JSON-shaped values, optional forms, and `CodeModeStringEnum` types. Throw `CodeModeFunctionError` for structured failures that should surface as CodeMode bridge errors. + +An enum-typed argument advertises its accepted values, so the catalog, the +generated TypeScript declaration, and argument validation all agree: + +```swift +enum Priority: String, CodeModeStringEnum { + case low, normal, high +} + +struct Arguments { + @CodeModeParam("Task identifier") + var id: String + @CodeModeParam("low, normal (default), or high") + var priority: Priority? +} +``` + +Providers do not have to be known at construction. `CodeModeAgentTools.register(provider:)` +adds them later — search advertises them immediately and the JavaScript bindings +are installed for the next execution — which is what a host needs when its domain +APIs depend on runtime state such as a signed-in account. diff --git a/Sources/CodeModeEvaluation/EvalModels.swift b/Sources/CodeModeEvaluation/EvalModels.swift index b8ae289..5511a4b 100644 --- a/Sources/CodeModeEvaluation/EvalModels.swift +++ b/Sources/CodeModeEvaluation/EvalModels.swift @@ -54,15 +54,39 @@ public struct CodeModeEvalExecuteStep: Codable, Sendable, Equatable { public var code: String public var allowedCapabilities: [CapabilityID] public var timeoutMs: Int? + /// Whether this step is *supposed* to fail. + /// + /// Repair scenarios deliberately call a helper wrong and then fix it from the + /// structured error, so their first step failing is the point. Every other + /// step failing should stop the run — otherwise a later step masks it and the + /// scenario passes on a broken transcript. + public var expectsFailure: Bool public init( code: String, allowedCapabilities: [CapabilityID] = [], - timeoutMs: Int? = nil + timeoutMs: Int? = nil, + expectsFailure: Bool = false ) { self.code = code self.allowedCapabilities = allowedCapabilities self.timeoutMs = timeoutMs + self.expectsFailure = expectsFailure + } + + private enum CodingKeys: String, CodingKey { + case code + case allowedCapabilities + case timeoutMs + case expectsFailure + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.code = try container.decode(String.self, forKey: .code) + self.allowedCapabilities = try container.decodeIfPresent([CapabilityID].self, forKey: .allowedCapabilities) ?? [] + self.timeoutMs = try container.decodeIfPresent(Int.self, forKey: .timeoutMs) + self.expectsFailure = try container.decodeIfPresent(Bool.self, forKey: .expectsFailure) ?? false } } diff --git a/Sources/CodeModeEvaluation/EvalRunner.swift b/Sources/CodeModeEvaluation/EvalRunner.swift index 30fba8a..2204a7f 100644 --- a/Sources/CodeModeEvaluation/EvalRunner.swift +++ b/Sources/CodeModeEvaluation/EvalRunner.swift @@ -58,6 +58,11 @@ public final class CodeModeEvalRunner: Sendable { searchDiagnostics = response.diagnostics } + // Logs and diagnostics accumulate across steps and an intermediate + // error stops the run. Overwriting each field per step meant + // validation only ever saw the last step: a scenario could pass while + // step 1 failed in a way step 2 masked, and step 1's logs — the + // evidence for why — were discarded. for executeStep in executeSteps(for: scenario) { toolCalls.append( CodeModeEvalToolCall( @@ -74,15 +79,25 @@ public final class CodeModeEvalRunner: Sendable { ) ) let observed = await observe(call) + // The graded value is the last step's output, and the graded error + // is the last step's error — so a successful repair step clears the + // deliberate failure that preceded it. executionOutput = observed.output - executionLogs = observed.logs - executionDiagnostics = observed.diagnostics observedError = observed.error + executionLogs.append(contentsOf: observed.logs) + executionDiagnostics.append(contentsOf: observed.diagnostics) + + // A repair scenario's first step is meant to fail and the next step + // fixes it. Anything else stops here rather than letting a later + // step mask the failure. + if observed.error != nil, executeStep.expectsFailure == false { + break + } } } catch let error as CodeModeToolError { observedError = error - executionLogs = error.logs - executionDiagnostics = error.diagnostics + executionLogs.append(contentsOf: error.logs) + executionDiagnostics.append(contentsOf: error.diagnostics) } catch { failures.append("Unexpected runner failure: \(error.localizedDescription)") } diff --git a/Sources/CodeModeEvaluation/EvalScenarios.swift b/Sources/CodeModeEvaluation/EvalScenarios.swift index 091e965..44359cb 100644 --- a/Sources/CodeModeEvaluation/EvalScenarios.swift +++ b/Sources/CodeModeEvaluation/EvalScenarios.swift @@ -16,6 +16,9 @@ public enum CodeModeEvalScenarios { filesystemCapabilityDenied, executionConsoleLogs, executionTimeout, + executionUnsettleablePromise, + executionTimerBackoff, + filesystemWholeJobInOneScript, reminderCatalogDiscovery, catalogFileSystemReadShape, catalogConsoleDiagnostics, @@ -319,10 +322,16 @@ public enum CodeModeEvalScenarios { return api.byJSName["apple.fs.read"]; } """, - executeCode: """ - return await apple.fs.read({ encoding: "utf8" }); - """, executeSteps: [ + // Deliberately wrong, so the next step can repair from the structured + // error. Marked so the runner does not treat it as the run failing. + CodeModeEvalExecuteStep( + code: """ + return await apple.fs.read({ encoding: "utf8" }); + """, + allowedCapabilities: [.fsRead], + expectsFailure: true + ), CodeModeEvalExecuteStep( code: """ const result = await apple.fs.read({ path: "tmp:repair.txt", encoding: "utf8" }); @@ -485,10 +494,10 @@ public enum CodeModeEvalScenarios { ) ) - public static let executionTimeout = CodeModeEvalScenario( - id: "execution.timeout", - title: "Unresolved promises time out", - task: "Await a never-resolving JavaScript promise. Do not catch the error in JavaScript; let executeJavaScript surface the structured execution-timeout error.", + public static let executionUnsettleablePromise = CodeModeEvalScenario( + id: "execution.unsettleable-promise", + title: "Unresolvable promises are reported, not waited out", + task: "Await a never-resolving JavaScript promise. Do not catch the error in JavaScript; let executeJavaScript surface the structured error.", executeCode: """ await new Promise(() => {}); return { never: true }; @@ -499,10 +508,93 @@ public enum CodeModeEvalScenarios { toolOrder: [.executeJavaScript], exactAllowedCapabilities: [], requiredExecuteCodeFragments: ["new Promise"], + expectedErrorCode: "JS_RUNTIME_ERROR" + ) + ) + + public static let executionTimeout = CodeModeEvalScenario( + id: "execution.timeout", + title: "CPU-bound scripts hit the execution timeout", + task: "Run a CPU-bound infinite loop. Do not catch the error in JavaScript; let executeJavaScript surface the structured execution-timeout error.", + executeCode: """ + while (true) {} + """, + allowedCapabilities: [], + timeoutMs: 200, + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript], + exactAllowedCapabilities: [], + requiredExecuteCodeFragments: ["while"], expectedErrorCode: "EXECUTION_TIMEOUT" ) ) + public static let filesystemWholeJobInOneScript = CodeModeEvalScenario( + id: "fs.whole-job-one-script", + title: "A multi-step job runs as one script", + task: "Read every .json receipt in documents:receipts, total the amounts for trip 'lisbon', and return { total, count }. Do the whole job in a single executeJavaScript call — list, read, filter, and sum inside the script — and return only the totals, not the receipts.", + searchCode: """ + async () => { + return api.references + .filter(ref => ["fs.list", "fs.read"].includes(ref.capability)) + .map(ref => ref.dts) + .join("\\n\\n"); + } + """, + executeCode: """ + const entries = await apple.fs.list({ path: 'documents:receipts' }); + let total = 0; + let count = 0; + for (const entry of entries) { + if (entry.isDirectory || !entry.name.endsWith('.json')) continue; + const { text } = await apple.fs.read({ path: entry.path }); + const receipt = JSON.parse(text); + if (receipt.trip !== 'lisbon') continue; + total += receipt.amount; + count += 1; + } + return { total, count }; + """, + allowedCapabilities: [.fsList, .fsRead], + seedFiles: [ + CodeModeEvalSeedFile(path: "documents:receipts/a.json", text: #"{"trip":"lisbon","amount":12}"#), + CodeModeEvalSeedFile(path: "documents:receipts/b.json", text: #"{"trip":"porto","amount":99}"#), + CodeModeEvalSeedFile(path: "documents:receipts/c.json", text: #"{"trip":"lisbon","amount":30}"#), + ], + expectation: CodeModeEvalExpectation( + // One search, one execute. A transcript that splits the list/read/sum + // across several executions fails here — that is the whole point of + // the scenario, and it only has teeth against real LLM transcripts. + toolOrder: [.searchJavaScriptAPI, .executeJavaScript], + exactAllowedCapabilities: [.fsList, .fsRead], + forbiddenCapabilities: [.fsWrite, .fsDelete, .fsMove], + requiredExecuteCodeFragments: ["apple.fs.list", "apple.fs.read"], + expectedOutput: .object(["total": .number(42), "count": .number(2)]) + ) + ) + + public static let executionTimerBackoff = CodeModeEvalScenario( + id: "execution.timer-backoff", + title: "setTimeout honours its delay", + task: "Use setTimeout to wait before returning, and cancel a second timer with clearTimeout so its callback never runs.", + executeCode: """ + const marks = []; + const cancelled = setTimeout(() => { marks.push('cancelled'); }, 5); + clearTimeout(cancelled); + await new Promise(resolve => setTimeout(resolve, 20)); + marks.push('resumed'); + return { marks }; + """, + allowedCapabilities: [], + timeoutMs: 2_000, + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript], + exactAllowedCapabilities: [], + requiredExecuteCodeFragments: ["setTimeout"], + expectedOutput: .object(["marks": .array([.string("resumed")])]) + ) + ) + public static let reminderCatalogDiscovery = CodeModeEvalScenario( id: "catalog.reminder-create", title: "Reminder helper discovery", diff --git a/Sources/CodeModeMacros/CodeModeMacro.swift b/Sources/CodeModeMacros/CodeModeMacro.swift index 72f6086..6d19b3f 100644 --- a/Sources/CodeModeMacros/CodeModeMacro.swift +++ b/Sources/CodeModeMacros/CodeModeMacro.swift @@ -116,12 +116,13 @@ public struct CodeModeMacro: ExtensionMacro { jsPath: \(literal(path)), title: \(literal(title(for: path))), summary: \(literal(summary)), - tags: [\(literal(parentPath(for: path)))], - example: "await \(path)({})", + tags: \(tagsSource(for: path)), + example: \(literal(example(for: path, fields: argumentFields.fields))), requiredArguments: [\(argumentFields.fields.filter { !$0.optional }.map { literal($0.name) }.joined(separator: ", "))], optionalArguments: [\(argumentFields.fields.filter(\.optional).map { literal($0.name) }.joined(separator: ", "))], argumentTypes: \(argumentTypesSource(for: argumentFields.fields)), argumentHints: \(argumentHintsSource(for: argumentFields.fields)), + argumentConstraints: \(argumentConstraintsSource(for: argumentFields.fields)), resultSummary: \(literal(returnsVoid ? "null" : resultInfo?.summary ?? "JSON value")), handler: { arguments, _ in try CodeModeAsyncBridge.run { @@ -275,7 +276,8 @@ public struct CodeModeMacro: ExtensionMacro { swiftType: typeInfo.swiftType, argumentType: typeInfo.argumentType, optional: typeInfo.optional, - description: attributeName.flatMap { attributeString(named: $0, attributes: variable.attributes).first } + description: attributeName.flatMap { attributeString(named: $0, attributes: variable.attributes).first }, + isStringEnum: typeInfo.isStringEnum ) ) } @@ -471,6 +473,60 @@ public struct CodeModeMacro: ExtensionMacro { return segments.dropLast().joined(separator: ".") } + /// Every path segment, so a helper is findable by its namespace *and* its + /// domain — the old single parent-path tag made `myapp.tasks.complete` + /// discoverable only by searching for "myapp.tasks". + private static func tagsSource(for path: String) -> String { + let segments = path.split(separator: ".").map(String.init) + var tags: [String] = [] + for segment in segments where tags.contains(segment) == false { + tags.append(segment) + } + if segments.count > 1 { + tags.append(segments.dropLast().joined(separator: ".")) + } + return "[" + tags.map(literal).joined(separator: ", ") + "]" + } + + /// A call the model can pattern-match, rather than the vacuous + /// `await path({})` that was emitted regardless of the arguments. + private static func example(for path: String, fields: [ToolField]) -> String { + let required = fields.filter { $0.optional == false } + guard required.isEmpty == false else { + return "await \(path)({})" + } + let pairs = required.map { field in + "\(field.name): \(placeholder(for: field))" + } + return "await \(path)({ " + pairs.joined(separator: ", ") + " })" + } + + private static func placeholder(for field: ToolField) -> String { + switch field.argumentType { + case "number": + return "0" + case "bool": + return "true" + case "array": + return "[]" + case "object": + return "{}" + default: + return "\"\(field.name)\"" + } + } + + private static func argumentConstraintsSource(for fields: [ToolField]) -> String { + let enumFields = fields.filter(\.isStringEnum) + guard enumFields.isEmpty == false else { + return "CapabilityArgumentConstraints.none" + } + let entries = enumFields.map { field in + "\(literal(field.name)): \(field.swiftType).codeModeAllowedValues" + }.joined(separator: ", ") + return "CapabilityArgumentConstraints(allowedStringValues: [\(entries)])" + } + private static func literal(_ value: String) -> String { let escaped = value .replacingOccurrences(of: "\\", with: "\\\\") @@ -511,6 +567,7 @@ private struct ToolField { var argumentType: String var optional: Bool var description: String? + var isStringEnum = false } private struct TypeInfo { @@ -518,6 +575,7 @@ private struct TypeInfo { var argumentType: String var optional: Bool var isSupported: Bool + var isStringEnum = false init(typeSyntax rawType: String) { let trimmed = rawType.replacingOccurrences(of: " ", with: "") @@ -554,6 +612,15 @@ private struct TypeInfo { if unwrapped.hasPrefix("["), unwrapped.hasSuffix("]") { self.argumentType = unwrapped.contains(":") ? "object" : "array" self.isSupported = Self.isSupportedCollection(unwrapped) + } else if Self.isSimpleIdentifier(unwrapped), Self.knownNonEnumTypes.contains(unwrapped) == false { + // An unrecognized simple type is taken to be a + // `CodeModeStringEnum`, the same assumption @BuiltInCodeMode + // makes. Host providers could not express constrained values at + // all before this, so their `allowedStringValues` was always + // empty while built-ins advertised theirs. + self.argumentType = "string" + self.isStringEnum = true + self.isSupported = true } else { self.argumentType = "any" self.isSupported = false @@ -561,6 +628,23 @@ private struct TypeInfo { } } + /// Types that are plainly not `CodeModeStringEnum`s. Without this list a + /// natural mistake such as `var date: Date` would be taken for an enum and + /// surface as a confusing error inside generated code instead of a macro + /// diagnostic pointing at the property. + private static let knownNonEnumTypes: Set = [ + "Date", "URL", "Data", "UUID", "Decimal", "Any", "AnyObject", "AnyHashable", + "Never", "Void", "Character", "Substring", "Int8", "Int16", "Int32", "Int64", + "UInt", "UInt8", "UInt16", "UInt32", "UInt64", "CGFloat", "NSNumber", "NSString", + ] + + private static func isSimpleIdentifier(_ type: String) -> Bool { + guard let first = type.first, first.isLetter || first == "_" else { + return false + } + return type.allSatisfy { $0.isLetter || $0.isNumber || $0 == "_" || $0 == "." } + } + private static func isSupportedCollection(_ type: String) -> Bool { let scalarTypes: Set = ["String", "Bool", "Int", "Double", "Float", "JSONValue"] let inner = String(type.dropFirst().dropLast()) diff --git a/TODO.md b/TODO.md index 282f76a..8a3b8e6 100644 --- a/TODO.md +++ b/TODO.md @@ -37,9 +37,11 @@ just the watchdog: serialization, cancellation, catch-proof termination, context recovery). - [ ] **JS heap / memory cap** — nothing bounds `JSContext`/`JSContextGroup` heap; `new Array(1e9)` can still exhaust host memory. NOT addressed. -- [ ] **Bound on concurrent executions** — `executionQueue` is `.concurrent` - with spin-waiting workers (`BridgeRuntime.swift:25-29`), so N long-running - scripts occupy N threads. No concurrency cap. NOT addressed. +- [x] **Bound on concurrent executions** — `ExecutionLimits.maxConcurrentExecutions` + (default 8) gates `runOnExecutionQueue` with a semaphore acquired *on* the + worker, so excess executions queue instead of exhausting the GCD thread pool. + Covered by `concurrentExecutionsAllCompleteAndStayIsolated` and + `executionsBeyondTheSlotLimitQueueRatherThanFail`. - [ ] `runOnExecutionQueue` still has no task-cancellation handler wired into the dispatched block (best-effort only). NOT addressed. @@ -238,6 +240,83 @@ tool model). → "resource- and network-bounded runtime." 3. Metadata-consolidation refactor + macro decision, protected by new drift tests. +## REVIEW FOLLOW-UP — STILL OPEN +From `REVIEW.md`. Everything in P0 and P2 is done; these are the parts of the P1 +items that were not completed, kept explicit so they do not read as finished. + +### #9 — async bridge ABI (partially done) +Done: real timer queue (`setTimeout`/`clearTimeout` honour delays and cancel), +immediate diagnosis of unsettleable promises, `BRIDGE_FAILURES_NOT_SURFACED` for +swallowed rejections, bounded concurrent executions, `ContinuousClock` deadlines, +deinit cancellation. +- [ ] **`CapabilityHandler` is still synchronous**, so `Promise.all([fetch(a), fetch(b)])` + is still serial. This is the change that requires resolving JS promises from + Swift via retained resolve/reject `JSValue`s on a per-execution serial executor. +- [ ] **Timeout/cancel still cannot interrupt an in-flight native call.** The + watchdog only traps while JS runs; `MediaBridge.transcode` can still overrun a + short `timeoutMs` by minutes. +- [ ] **`CodeModeAsyncBridge.run` still parks a semaphore** for up to 30s (longer + than the 10s default execution timeout) for async host tools. +- [ ] Optional: pool `JSVirtualMachine`s. Constraint to respect — the JSC time + limit is per context *group*, so a pooled VM must never host two concurrent + executions. + +### #11 — generate the JS bindings from the catalog +Done: the concrete drift found while generating declarations is fixed (keychain +object/positional, `fs.promises.rename`/`copyFile` options, location argument +pass-through, tool-description calling conventions). +- [ ] `RuntimeJavaScript.bootstrap` is still ~250 hand-written lines that can + drift from the catalog again. Wrappers that inject hidden arguments + (`createEvent` forces `operation: 'create'`) still cannot be expressed in + catalog metadata, so the generated `.d.ts` advertises an argument the wrapper + overrides. + +### #13 — provider ergonomics + MCP +Done: catalog unfrozen (post-init `register(provider:)` works and stays in sync), +enum constraints, real examples, per-segment tags. +- [ ] `@CodeModeProvider(namespace:)` on a type with `@CodeModeTool` on methods — + today it is still one struct per JavaScript function. +- [ ] `requiredPermissions` and typed results for provider registrations. +- [ ] MCP adapter materializing an MCP server's tools as `CodeModeRegistration`s + under `mcp..`. Blocked on async handlers (#9). + +### Test seams +Done: EventKit event/reminder serialization is now covered directly (the two +implicit-unwrap crashes), concurrent execution is covered, CI runners are pinned +to `macos-15` rather than `macos-latest`. `FetchTaskHandler`'s redirect +re-validation and size-cap cancellation turned out to be covered already +(`NetworkAccessPolicyTests`), contrary to the review. +- [ ] **Store protocols for EventKit/Health/Photos/Home/Contacts**, mirroring + `CodeModeFileSystem`. Their create/update/filter logic still executes nowhere: + the bridges hard-instantiate `EKEventStore()`, `HKHealthStore()`, `PHAsset`, so + tests can only reach permission denial and argument validation. +- [ ] A simulator test job, so UIKit presenters, AlarmKit, and the TCC-gated happy + paths are executed somewhere rather than only compiled. + +### Smaller P2 items not taken +- [ ] **No "prompt pending" permission status.** `request*` waits 10s and ignores + the timeout, so a user who takes 11s gets `PERMISSION_DENIED` while the dialog + is still on screen. Needs a new `PermissionStatus` case, which is source-breaking + for host code switching exhaustively. + +## DECLINED REVIEW FINDINGS +- **"Stop imposing swift-syntax on every consumer" (REVIEW.md P1 #12)** — + considered 2026-07-25 and declined. The proposal is to check in the expanded + `@BuiltInCodeMode` members so the `CodeMode` target can drop its + `CodeModeMacros` dependency. Declined because: + - The cost it cites is already measured and mitigated: prebuilt swift-syntax is + default-on for macros on the supported toolchains (`PLAN-registration-macros.md` + §"The swift-syntax cost, measured today"), so consumers pay ~0 incremental + build time, and the version range is already widened to `602.0.0..<604.0.0` + for `mlx-swift-lm` co-resolution — the one concrete conflict named. + - The migration is effectively one-way across 117 tool definitions and strips + the `@ToolParam("hint")` annotations off the property declarations, leaving + the hints readable only inside the generated argument arrays. + - It works against REVIEW #11 and #13, which both ask for *more* generation + from this metadata, not less. + Revisit if a consumer hits a swift-syntax version conflict the range cannot + absorb, or if prebuilts stop being default-on. + ## OPEN QUESTIONS FOR THE USER - [ ] Decision on the private-but-exported JSC symbol (App Store risk)? - [ ] After CI is green: tag `0.1.0` and restore the versioned install snippet? diff --git a/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift b/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift index a6bed25..513097c 100644 --- a/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift +++ b/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift @@ -69,7 +69,7 @@ private struct FailingTaskTool: CodeModeProvider { #expect(registration.jsPath == "myapp.tasks.complete") #expect(registration.title == "complete") #expect(registration.summary == "Mark a task complete.") - #expect(registration.tags == ["myapp.tasks"]) + #expect(registration.tags == ["myapp", "tasks", "complete", "myapp.tasks"]) #expect(registration.requiredArguments == ["id"]) #expect(registration.optionalArguments == ["note"]) #expect(registration.argumentTypes["id"] == .string) @@ -233,12 +233,13 @@ private func observe(_ call: JavaScriptExecutionCall) async -> ObservedExecution jsPath: "macro.api.greet", title: "greet", summary: "Greet a person.", - tags: ["macro.api"], - example: "await macro.api.greet({})", + tags: ["macro", "api", "greet", "macro.api"], + example: "await macro.api.greet({ name: \\"name\\" })", requiredArguments: ["name"], optionalArguments: ["excited"], argumentTypes: ["name": CapabilityArgumentType.string, "excited": CapabilityArgumentType.bool], argumentHints: ["name": "Person name", "excited": "Whether to add emphasis"], + argumentConstraints: CapabilityArgumentConstraints.none, resultSummary: "Greeting payload", handler: { arguments, _ in try CodeModeAsyncBridge.run { @@ -292,12 +293,13 @@ private func observe(_ call: JavaScriptExecutionCall) async -> ObservedExecution jsPath: "macro.api.ping", title: "ping", summary: "Ping.", - tags: ["macro.api"], + tags: ["macro", "api", "ping", "macro.api"], example: "await macro.api.ping({})", requiredArguments: [], optionalArguments: [], argumentTypes: [:], argumentHints: [:], + argumentConstraints: CapabilityArgumentConstraints.none, resultSummary: "null", handler: { arguments, _ in try CodeModeAsyncBridge.run { @@ -511,3 +513,76 @@ private func observe(_ call: JavaScriptExecutionCall) async -> ObservedExecution macros: codeModeTestMacros() ) } + +// MARK: - Constrained argument values for host providers + +/// Host providers could not express constrained values at all: the macro +/// rejected any non-primitive property type, so their `allowedStringValues` was +/// always empty while built-ins advertised theirs. This is the parity fix. +enum DemoPriority: String, CodeModeStringEnum { + case low + case normal + case high +} + +@CodeMode(path: "macro.api.prioritize", description: "Set a priority.") +struct PrioritizeTool: Sendable { + struct Arguments { + @CodeModeParam("Task identifier") + var id: String + @CodeModeParam("low, normal (default), or high") + var priority: DemoPriority? + } + + struct Result { + var id: String + var priority: String + } + + func call(arguments: Arguments) throws -> Result { + Result(id: arguments.id, priority: (arguments.priority ?? .normal).rawValue) + } +} + +@Test func codeModeMacroAdvertisesEnumConstraintsForHostProviders() throws { + let registration = try #require(PrioritizeTool().codeModeRegistrations().first) + + #expect(registration.argumentTypes["priority"] == .string) + #expect(registration.argumentConstraints.allowedStringValues["priority"] == ["low", "normal", "high"]) + // The generated example is a call the model can pattern-match, not the + // vacuous `await path({})` that used to be emitted regardless of arguments. + #expect(registration.example == #"await macro.api.prioritize({ id: "id" })"#) + // Tags cover every path segment, so the helper is findable by namespace and + // by name, not only by the full parent path. + #expect(registration.tags.contains("prioritize")) + #expect(registration.tags.contains("macro.api")) +} + +@Test func codeModeMacroRejectsValuesOutsideTheConstraint() async throws { + let tools = CodeModeAgentTools(config: CodeModeConfiguration(codeModeProviders: [PrioritizeTool()])) + + let denied = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await macro.api.prioritize({ id: "1", priority: "urgent" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["macro.api.prioritize"] + ) + ) + var deniedError: CodeModeToolError? + do { + _ = try await denied.result + } catch let error as CodeModeToolError { + deniedError = error + } + #expect(deniedError?.code == "INVALID_ARGUMENTS") + + let accepted = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await macro.api.prioritize({ id: "1", priority: "high" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["macro.api.prioritize"] + ) + ) + let result = try await accepted.result + #expect(result.output?.objectValue?["priority"] == .string("high")) +} diff --git a/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift b/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift index 340f01b..5580d09 100644 --- a/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift +++ b/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift @@ -142,3 +142,147 @@ private func failureSummary(_ results: [CodeModeEvalResult]) -> String { } .joined(separator: "\n") } + +// MARK: - Multi-step scenarios + +// The runner overwrote executionOutput/Logs/Diagnostics/observedError on every +// step and did not stop on an intermediate error, so validation saw only the last +// step: a scenario could pass while step 1 failed in a way step 2 masked, and +// step 1's logs — the evidence — were gone. + +@Test func multiStepScenariosAccumulateLogsAcrossSteps() async throws { + let scenario = CodeModeEvalScenario( + id: "multi-step-log-accumulation", + title: "Logs from every step survive", + task: "internal test", + executeSteps: [ + CodeModeEvalExecuteStep(code: "console.log('step-one-ran'); return 1;", allowedCapabilities: []), + CodeModeEvalExecuteStep(code: "console.log('step-two-ran'); return 2;", allowedCapabilities: []), + ], + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript, .executeJavaScript], + exactAllowedCapabilities: [], + requiredExecutionLogFragments: ["step-one-ran", "step-two-ran"], + expectedOutput: .number(2) + ) + ) + + let result = await CodeModeEvalRunner().run(scenario) + #expect(result.passed, "\(result.failures)") + #expect(result.executionLogs.contains { $0.message.contains("step-one-ran") }) + #expect(result.executionLogs.contains { $0.message.contains("step-two-ran") }) + #expect(result.executionOutput == .number(2)) +} + +@Test func anIntermediateStepFailureStopsTheRunAndIsReported() async throws { + let scenario = CodeModeEvalScenario( + id: "multi-step-intermediate-failure", + title: "A failing first step is not masked by a passing second", + task: "internal test", + executeSteps: [ + // Denied: the step declares no capabilities. + CodeModeEvalExecuteStep(code: "return await apple.fs.read({ path: 'tmp:nope.txt' });", allowedCapabilities: []), + CodeModeEvalExecuteStep(code: "return 'second step succeeded';", allowedCapabilities: []), + ], + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript], + exactAllowedCapabilities: [], + expectedErrorCode: "CAPABILITY_DENIED" + ) + ) + + let result = await CodeModeEvalRunner().run(scenario) + #expect(result.error?.code == "CAPABILITY_DENIED") + // The second step must not have run and must not have supplied the output. + #expect(result.toolCalls.filter { $0.tool == .executeJavaScript }.count == 1) + #expect(result.executionOutput != .string("second step succeeded")) +} + +// MARK: - Grading the one-script-per-job standard + +// The docs now tell agents to do a whole job in one executeJavaScript call. That +// is only a real standard if a transcript that ignores it fails, so this grades +// the shape rather than just asserting the reference solution. + +@Test func aTranscriptSplitAcrossExecutionsFailsTheOneScriptScenario() { + let scenario = CodeModeEvalScenarios.filesystemWholeJobInOneScript + + // The anti-pattern: list in one call, read in another, sum in a third — + // every intermediate result round-tripped through the model's context. + let split = [ + CodeModeEvalToolCall(tool: .searchJavaScriptAPI, code: scenario.searchCode ?? ""), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: "return await apple.fs.list({ path: 'documents:receipts' });", + allowedCapabilities: [.fsList] + ), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: "return await apple.fs.read({ path: 'documents:receipts/a.json' });", + allowedCapabilities: [.fsRead] + ), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: "return await apple.fs.read({ path: 'documents:receipts/c.json' });", + allowedCapabilities: [.fsRead] + ), + ] + + let failures = CodeModeEvalRunner().validateTranscript( + scenario: scenario, + toolCalls: split, + searchResult: .string("fs.list fs.read"), + executionOutput: .object(["total": .number(42), "count": .number(2)]), + error: nil + ) + + #expect(failures.contains { $0.contains("Tool order") }) +} + +@Test func theSingleScriptTranscriptPassesTheOneScriptScenario() { + let scenario = CodeModeEvalScenarios.filesystemWholeJobInOneScript + + let single = [ + CodeModeEvalToolCall(tool: .searchJavaScriptAPI, code: scenario.searchCode ?? ""), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: scenario.executeCode ?? "", + allowedCapabilities: [.fsList, .fsRead] + ), + ] + + let failures = CodeModeEvalRunner().validateTranscript( + scenario: scenario, + toolCalls: single, + searchResult: .string("fs.list fs.read"), + executionOutput: .object(["total": .number(42), "count": .number(2)]), + error: nil + ) + + #expect(failures.isEmpty, "\(failures)") +} + +@Test func anOverBroadCapabilityRequestFailsEvenWithTheRightAnswer() { + let scenario = CodeModeEvalScenarios.filesystemWholeJobInOneScript + + // Minimization is now stated plainly in the tool description, so grading it + // is no longer an unstated rule. + let overBroad = [ + CodeModeEvalToolCall(tool: .searchJavaScriptAPI, code: scenario.searchCode ?? ""), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: scenario.executeCode ?? "", + allowedCapabilities: [.fsList, .fsRead, .fsWrite] + ), + ] + + let failures = CodeModeEvalRunner().validateTranscript( + scenario: scenario, + toolCalls: overBroad, + searchResult: .string("fs.list fs.read"), + executionOutput: .object(["total": .number(42), "count": .number(2)]), + error: nil + ) + + #expect(failures.contains { $0.contains("Allowed capabilities") || $0.contains("Forbidden capabilities") }) +} diff --git a/Tests/CodeModeTests/AgentGuidanceTests.swift b/Tests/CodeModeTests/AgentGuidanceTests.swift new file mode 100644 index 0000000..d3bf642 --- /dev/null +++ b/Tests/CodeModeTests/AgentGuidanceTests.swift @@ -0,0 +1,157 @@ +import Foundation +import Testing +@testable import CodeMode + +// The tool descriptions and the generated TypeScript teach an agent what the +// helpers are called. Nothing taught it what shape its work should take, so an +// agent could reasonably use executeJavaScript as a thin RPC — one call per +// operation — which is the pattern this package exists to replace. + +@Test func everyTierStatesTheCoreIdea() { + // A host dropping to a smaller tier must not lose a rule, only the examples. + for length in CodeModeAgentGuidance.Length.allCases { + let text = CodeModeAgentGuidance.systemPrompt(length) + #expect(text.contains("One `executeJavaScript` call should do the entire job")) + #expect(text.contains("Request only the capabilities your script actually calls")) + } +} + +@Test func tiersGrowStrictlyAndAreAdditive() { + let brief = CodeModeAgentGuidance.systemPrompt(.brief) + let standard = CodeModeAgentGuidance.systemPrompt(.standard) + let full = CodeModeAgentGuidance.systemPrompt(.full) + + #expect(brief.count < standard.count) + #expect(standard.count < full.count) + // Additive, not rewritten: the larger tiers contain the smaller ones verbatim. + #expect(standard.contains(brief)) + #expect(full.contains(standard)) +} + +@Test func largerTiersAddExamplesAndFailureModes() { + let standard = CodeModeAgentGuidance.systemPrompt(.standard) + let full = CodeModeAgentGuidance.systemPrompt(.full) + + #expect(standard.contains("```javascript")) + #expect(standard.contains("searchJavaScriptAPI")) + #expect(full.contains("Anti-patterns")) + #expect(full.contains("BRIDGE_FAILURES_NOT_SURFACED")) + // The concurrency reality must not be over-promised in the fan-out example. + #expect(full.contains("`Promise.all` does not make these overlap")) +} + +@Test func aBudgetSelectsTheLargestTierThatFits() { + let brief = CodeModeAgentGuidance.Length.brief.approximateTokenCount + let standard = CodeModeAgentGuidance.Length.standard.approximateTokenCount + let full = CodeModeAgentGuidance.Length.full.approximateTokenCount + + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: full) == .full) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: full + 5_000) == .full) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: full - 1) == .standard) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: standard) == .standard) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: standard - 1) == .brief) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: brief) == .brief) +} + +@Test func aBudgetTooSmallForAnythingStillReturnsTheCoreIdea() { + // Deliberately a floor rather than an empty string: some guidance is the + // difference between an agent that batches and one that does not. + #expect(CodeModeAgentGuidance.systemPrompt(approximateTokenBudget: 0) == CodeModeAgentGuidance.systemPrompt(.brief)) + #expect(CodeModeAgentGuidance.systemPrompt(approximateTokenBudget: -100) == CodeModeAgentGuidance.systemPrompt(.brief)) +} + +@Test func guidanceExamplesUseHelpersAndCapabilitiesThatActuallyExist() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let full = CodeModeAgentGuidance.systemPrompt(.full) + let jsNames = Set(tools.capabilities().flatMap(\.jsNames)) + + // Guidance naming a helper the runtime does not install would teach the exact + // failure it is trying to prevent. + for helper in ["apple.fs.list", "apple.fs.read", "apple.fs.move"] { + #expect(full.contains(helper)) + #expect(jsNames.contains(helper), "guidance references missing helper \(helper)") + } + + for capability in ["fs.list", "fs.read", "fs.move"] { + #expect(CapabilityID(rawValue: capability) != nil, "guidance references missing capability \(capability)") + } +} + +@Test func theWorkedExampleActuallyRuns() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // The example is the shape we ask agents to copy, so it has to be a script + // this runtime accepts — arguments, calling conventions, and all. The setup + // lines are the only addition. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.fs.mkdir({ path: 'documents:receipts/lisbon' }); + await apple.fs.write({ + path: 'documents:receipts/a.json', + data: JSON.stringify({ trip: 'lisbon', amount: 12 }) + }); + await apple.fs.write({ + path: 'documents:receipts/b.json', + data: JSON.stringify({ trip: 'porto', amount: 99 }) + }); + + const entries = await apple.fs.list({ path: 'documents:receipts' }); + let total = 0; + const filed = []; + + for (const entry of entries) { + if (entry.isDirectory || !entry.name.endsWith('.json')) continue; + const { text } = await apple.fs.read({ path: entry.path }); + const receipt = JSON.parse(text); + if (receipt.trip !== 'lisbon') continue; + total += receipt.amount; + await apple.fs.move({ + from: entry.path, + to: `documents:receipts/lisbon/${entry.name}`, + overwrite: true + }); + filed.push(entry.name); + } + + return { total, filedCount: filed.length }; + """, + allowedCapabilities: [.fsMkdir, .fsWrite, .fsList, .fsRead, .fsMove] + ) + ) + + #expect(observed.error == nil) + let output = try #require(observed.result?.output?.objectValue) + #expect(output.int("total") == 12) + #expect(output.int("filedCount") == 1) +} + +@Test func toolDescriptionExamplesNameRealHelpersAndArguments() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let description = CodeModeAgentToolDescriptions.executeJavaScript.description + let references = tools.capabilities() + let byJSName = Dictionary(references.flatMap { ref in ref.jsNames.map { ($0, ref) } }, uniquingKeysWith: { first, _ in first }) + + // An example that names a helper or argument the runtime does not have + // teaches the exact failure the examples exist to prevent — and a wrong + // argument name is the easiest kind to get wrong when writing prose. + let used: [(helper: String, arguments: [String])] = [ + ("apple.calendar.listEvents", ["start", "end"]), + ("apple.contacts.list", ["identifiers"]), + ] + + for (helper, arguments) in used { + #expect(description.contains(helper)) + let reference = try #require(byJSName[helper], "description references missing helper \(helper)") + let known = Set(reference.requiredArguments + reference.optionalArguments) + for argument in arguments { + #expect(known.contains(argument), "\(helper) has no argument '\(argument)'") + } + } +} diff --git a/Tests/CodeModeTests/CodeModeProviderTests.swift b/Tests/CodeModeTests/CodeModeProviderTests.swift index 67387c9..e444951 100644 --- a/Tests/CodeModeTests/CodeModeProviderTests.swift +++ b/Tests/CodeModeTests/CodeModeProviderTests.swift @@ -578,7 +578,11 @@ private func jsonLiteral(_ value: JSONValue) -> String { #expect(result.bool("done") == true) } -@Test func builtInCapabilitiesCanBeAllowedByCapabilityKey() async throws { +// `allowedCapabilityKeys` accepts arbitrary strings and is not validated against +// `CapabilityID` at decode time, so it must never reach a built-in bridge — a host +// that vets only the strictly-typed `allowedCapabilities` would otherwise be +// bypassed by spelling the same ID in the loose field. +@Test func builtInCapabilitiesAreNotGrantedByCapabilityKey() async throws { let (tools, sandbox) = try makeTools() defer { cleanup(sandbox) } @@ -590,9 +594,78 @@ private func jsonLiteral(_ value: JSONValue) -> String { ) ) let observed = await observe(call) + let error = try #require(observed.error) + #expect(error.code == "CAPABILITY_DENIED") + #expect(error.capabilityKey == CapabilityID.keychainRead.codeModeKey) +} + +@Test func builtInCapabilitiesRemainGrantedByCapabilityID() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await apple.keychain.get("missing-provider-key-test");"#, + allowedCapabilities: [.keychainRead] + ) + ) + let observed = await observe(call) #expect(observed.result?.output == .null) } +@Test func hostCapabilityGrantOverridesModelAuthoredAllowlist() async throws { + let (tools, sandbox) = try makeTools( + capabilityGrant: .only([.fsRead], capabilityKeys: []), + codeModeProviders: [ManualProvider()] + ) + defer { cleanup(sandbox) } + + // The script asks for keychain.read and a provider key; the host grants neither. + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await apple.keychain.get("anything");"#, + allowedCapabilities: [.keychainRead], + allowedCapabilityKeys: ["myapp.api.doTheThing"] + ) + ) + let observed = await observe(call) + let error = try #require(observed.error) + #expect(error.code == "CAPABILITY_DENIED") + #expect(error.suggestions.contains { $0.contains("host app's capability grant") }) + #expect( + error.diagnostics.contains { $0.code == "CAPABILITY_WITHHELD_BY_HOST" } + ) + + let providerCall = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.api.doTheThing({ id: "123" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.api.doTheThing"] + ) + ) + let providerObserved = await observe(providerCall) + #expect(providerObserved.error?.code == "CAPABILITY_DENIED") +} + +@Test func hostCapabilityGrantPermitsWhatItIncludes() async throws { + let (tools, sandbox) = try makeTools( + capabilityGrant: .only([], capabilityKeys: ["myapp.api.doTheThing"]), + codeModeProviders: [ManualProvider()] + ) + defer { cleanup(sandbox) } + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.api.doTheThing({ id: "123" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.api.doTheThing"] + ) + ) + let observed = await observe(call) + let result = try #require(observed.result?.output?.objectValue) + #expect(result.string("id") == "123") +} + @Test func customProviderFunctionErrorsRemainStructured() async throws { let (tools, sandbox) = try makeTools(codeModeProviders: [ManualProvider()]) defer { cleanup(sandbox) } @@ -632,3 +705,78 @@ private func jsonLiteral(_ value: JSONValue) -> String { #expect(result.string("custom") == "myapp.api.doTheThing") #expect(result.string("builtIn") == CapabilityID.weatherRead.rawValue) } + +// MARK: - Post-init provider registration + +private struct LateProvider: CodeModeProvider { + let codeModePath = "myapp.late" + + func codeModeRegistrations() -> [CodeModeRegistration] { + [ + CodeModeRegistration( + capabilityKey: "myapp.late.echo", + jsPath: "myapp.late.echo", + title: "echo", + summary: "Echo a value registered after init.", + tags: ["myapp", "late"], + example: #"await myapp.late.echo({ value: "hi" })"#, + requiredArguments: ["value"], + argumentTypes: ["value": .string], + argumentHints: ["value": "Value to echo"], + resultSummary: "Echoed value" + ) { arguments, _ in + .object(["value": .string(try CodeModeArgumentDecoder.requireString("value", in: arguments))]) + }, + ] + } +} + +// The catalog used to be snapshotted in CodeModeAgentTools.init, so the +// registry's public post-init register(...) methods were unreachable in the +// supported flow — and would have desynced search from execution if reached. +@Test func providersRegisteredAfterInitAreExecutable() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let before = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.late.echo({ value: "hi" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.late.echo"] + ) + ) + // Not registered yet: the helper simply does not exist in the context. + let beforeObserved = await observe(before) + #expect(beforeObserved.error != nil) + + tools.register(provider: LateProvider()) + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.late.echo({ value: "hi" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.late.echo"] + ) + ) + let observed = await observe(call) + #expect(observed.result?.output?.objectValue?.string("value") == "hi") +} + +@Test func providersRegisteredAfterInitAppearInSearchAndTypeDeclarations() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + #expect(tools.capabilities().contains { $0.capability == "myapp.late.echo" } == false) + + tools.register(provider: LateProvider()) + + // Search must not advertise anything execution cannot invoke, and vice versa. + let reference = try #require(tools.capabilities().first { $0.capability == "myapp.late.echo" }) + #expect(reference.dts.contains("value: string;")) + #expect(tools.typeDeclarations().contains("namespace late {")) + + let response = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest(code: #"async () => { return api.byJSName["myapp.late.echo"].summary; }"#) + ) + #expect(response.result == .string("Echo a value registered after init.")) +} diff --git a/Tests/CodeModeTests/CompletionWaitTests.swift b/Tests/CodeModeTests/CompletionWaitTests.swift new file mode 100644 index 0000000..0c91fce --- /dev/null +++ b/Tests/CodeModeTests/CompletionWaitTests.swift @@ -0,0 +1,57 @@ +import Foundation +import Testing +@testable import CodeMode + +// The pattern this replaces — a `var` written from a framework's completion +// queue and read after a semaphore wait whose timeout result was discarded — is +// both a data race and an ambiguity: the caller could not tell "timed out" from +// "the API returned nothing". + +@Test func completionWaitReturnsTheDeliveredValue() throws { + // 60s, not 5: the suite runs in parallel and this runtime's timer waits block + // GCD workers with `Thread.sleep`, so even an immediate `global().async` can + // wait seconds for a thread on a loaded runner. The assertion is that the + // delivered value comes back, not how fast — and the thread pressure itself is + // a property of the synchronous bridge model that the async ABI (#9) removes. + let value = try CompletionWait.value(timeout: 60, operationName: "test") { complete in + DispatchQueue.global().async { complete([1, 2, 3]) } + } + #expect(value == [1, 2, 3]) +} + +@Test func completionWaitThrowsRatherThanReturningAnEmptyResult() { + do { + // Never completes: the old shape returned `.array([])`, indistinguishable + // from a genuinely empty fetch. + let value: [Int] = try CompletionWait.value(timeout: 0.1, operationName: "reminders.read") { _ in } + Issue.record("Expected a timeout, got \(value)") + } catch { + #expect(requireBridgeErrorCode(error) == "EXECUTION_TIMEOUT") + } +} + +@Test func completionWaitToleratesALateCallback() throws { + // The late write must land in the box rather than racing a caller's read. + let lateCallbackFinished = DispatchSemaphore(value: 0) + do { + let _: Int = try CompletionWait.value(timeout: 0.05, operationName: "test") { complete in + DispatchQueue.global().asyncAfter(deadline: .now() + 0.2) { + complete(7) + lateCallbackFinished.signal() + } + } + Issue.record("Expected a timeout") + } catch { + #expect(requireBridgeErrorCode(error) == "EXECUTION_TIMEOUT") + } + + // Generous on purpose: the suite runs in parallel and this runtime's timer + // waits block GCD workers, so a 0.2s dispatch can be starved for seconds on a + // loaded runner. The assertion is that the late callback lands at all. + #expect(lateCallbackFinished.wait(timeout: .now() + 60) == .success) +} + +@Test func completionWaitReportsWhetherTheCallbackArrived() { + #expect(CompletionWait.completion(timeout: 5) { complete in complete() }) + #expect(CompletionWait.completion(timeout: 0.05) { _ in } == false) +} diff --git a/Tests/CodeModeTests/DefensiveDecodingTests.swift b/Tests/CodeModeTests/DefensiveDecodingTests.swift index 8a4485c..e9ede14 100644 --- a/Tests/CodeModeTests/DefensiveDecodingTests.swift +++ b/Tests/CodeModeTests/DefensiveDecodingTests.swift @@ -56,6 +56,50 @@ private func decodeRequest(_ json: String) throws -> JavaScriptExecutionRequest } } +// MARK: - Adversarial numbers must not trap + +@Test func executionRequestRejectsNonRepresentableTimeout() throws { + // `Int.init(_: Double)` traps on these. This decode path runs on + // model-authored JSON before any script does, so a trap here is an + // in-process kill of the host app. + for literal in ["1e300", "-1e300", "1e400", "-1e400"] { + #expect(throws: (any Error).self) { + _ = try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":\#(literal)}"#) + } + #expect(throws: (any Error).self) { + _ = try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":"\#(literal)"}"#) + } + } + + for literal in ["inf", "-inf", "infinity", "nan", "NaN"] { + #expect(throws: (any Error).self) { + _ = try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":"\#(literal)"}"#) + } + } +} + +@Test func executionRequestClampsTimeoutToAdvertisedBounds() throws { + // The schema advertises 1...60000; out-of-range values are clamped rather + // than failing the call. + #expect(try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":600000}"#).timeoutMs == 60_000) + #expect(try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":0}"#).timeoutMs == 1) + #expect(try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":-5}"#).timeoutMs == 1) + #expect(JavaScriptExecutionRequest(code: "", allowedCapabilities: [], timeoutMs: 10_000_000).timeoutMs == 60_000) +} + +@Test func jsonValueIntConversionIsTotal() { + #expect(JSONValue.number(.infinity).intValue == nil) + #expect(JSONValue.number(-.infinity).intValue == nil) + #expect(JSONValue.number(.nan).intValue == nil) + #expect(JSONValue.number(1e300).intValue == nil) + #expect(JSONValue.number(-1e300).intValue == nil) + #expect(JSONValue.number(9.3e18).intValue == nil) + #expect(JSONValue.number(42).intValue == 42) + #expect(JSONValue.number(-42.9).intValue == -42) + #expect(JSONValue.number(Double(Int.max)).intValue == nil) + #expect(JSONValue.number(Double(Int.min)).intValue == Int.min) +} + @Test func executionRequestRoundTripsThroughCodable() throws { let original = JavaScriptExecutionRequest( code: "return 1;", @@ -118,6 +162,38 @@ private func makeEchoRegistry(argumentTypes: [String: CapabilityArgumentType]) - } } +@Test func registryRejectsNonFiniteDeclaredNumbers() throws { + let registry = makeEchoRegistry(argumentTypes: ["timeoutMs": .number]) + let (context, sandbox) = try makeInvocationContext(allowedCapabilities: [.fsRead]) + defer { cleanup(sandbox) } + + // Straight through as a JSON number... + #expect(throws: (any Error).self) { + _ = try registry.invoke( + CapabilityID.fsRead.rawValue, + arguments: ["timeoutMs": .number(.infinity)], + context: context + ) + } + // ...and smuggled through string coercion, which `Double.init(String)` accepts. + for spelling in ["inf", "-infinity", "nan"] { + #expect(throws: (any Error).self) { + _ = try registry.invoke( + CapabilityID.fsRead.rawValue, + arguments: ["timeoutMs": .string(spelling)], + context: context + ) + } + } + + let ok = try registry.invoke( + CapabilityID.fsRead.rawValue, + arguments: ["timeoutMs": .string("2500")], + context: context + ) + #expect(ok.objectValue?["timeoutMs"] == .number(2_500)) +} + @Test func registryDoesNotCoerceNonBooleanStringsOrBadNumbers() throws { let registry = makeEchoRegistry(argumentTypes: ["count": .number, "flag": .bool]) let (context, sandbox) = try makeInvocationContext(allowedCapabilities: [.fsRead]) diff --git a/Tests/CodeModeTests/EventKitBridgeTests.swift b/Tests/CodeModeTests/EventKitBridgeTests.swift index 6fb95b9..da6c32a 100644 --- a/Tests/CodeModeTests/EventKitBridgeTests.swift +++ b/Tests/CodeModeTests/EventKitBridgeTests.swift @@ -135,3 +135,44 @@ import Testing #expect(observed.error?.code == "PERMISSION_DENIED") } + +// MARK: - Serialization of degenerate items + +#if canImport(EventKit) +import EventKit + +// EventKit's serialization logic executes nowhere today — not in unit tests, not +// in CI, not in evals — because the bridge hard-instantiates EKEventStore and the +// tests can only reach permission denial. These construct the model objects +// directly, which needs no permission, and cover the two implicitly-unwrapped +// fields that crashed the execution thread. + +@Test func eventSerializationSurvivesAnEventWithNoCalendar() throws { + let store = EKEventStore() + let event = EKEvent(eventStore: store) + event.title = "Orphaned" + event.startDate = Date(timeIntervalSince1970: 1_700_000_000) + event.endDate = event.startDate.addingTimeInterval(3_600) + + // `EKEvent.calendar` is `EKCalendar!` and is nil for an orphaned event, while + // every neighbouring field was nil-coalesced. + #expect(event.calendar == nil) + + let json = try #require(EventKitBridge.eventJSONForTesting(event).objectValue) + #expect(json.string("title") == "Orphaned") + #expect(json.string("calendarIdentifier") == "") + #expect(json.string("calendarTitle") == "") + #expect(json.string("notes") == "") +} + +@Test func reminderSerializationSurvivesAMissingTitle() throws { + let store = EKEventStore() + let reminder = EKReminder(eventStore: store) + + // `EKReminder.title` is `String!`. + let json = try #require(EventKitBridge.reminderJSONForTesting(reminder).objectValue) + #expect(json.string("title") == "") + #expect(json.string("calendarIdentifier") == "") + #expect(json.bool("isCompleted") == false) +} +#endif diff --git a/Tests/CodeModeTests/EventLoopTests.swift b/Tests/CodeModeTests/EventLoopTests.swift new file mode 100644 index 0000000..3f2f0b1 --- /dev/null +++ b/Tests/CodeModeTests/EventLoopTests.swift @@ -0,0 +1,394 @@ +import Foundation +import Testing +@testable import CodeMode + +// `setTimeout` used to invoke its callback synchronously and ignore the delay. +// Three things were wrong at once: clearTimeout could never cancel, an error +// thrown by a callback propagated to setTimeout's *caller*, and +// `await new Promise(r => setTimeout(r, 2000))` — the standard backoff — was a +// hot loop that hammered remote APIs through fetch. + +@Test func setTimeoutDefersItsCallbackInsteadOfRunningItInline() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => { order.push('timer'); }, 0); + order.push('sync'); + await new Promise(resolve => setTimeout(resolve, 1)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.error == nil) + let order = try #require(observed.result?.output?.objectValue?.array("order")) + #expect(order == [.string("sync"), .string("timer")]) +} + +@Test func setTimeoutActuallyWaitsTheRequestedDelay() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 150)); return 'done';", + allowedCapabilities: [], + timeoutMs: 5_000 + ) + ) + let elapsed = Date().timeIntervalSince(started) + + #expect(observed.result?.output == .string("done")) + // The assertion is the lower bound: the delay was previously ignored + // entirely, so a backoff loop spun. There is deliberately no tight upper + // bound — a shared CI runner can stretch a 150ms wait by seconds, and the + // script's own 5s timeoutMs already fails the test if the wait runs away. + #expect(elapsed >= 0.14) +} + +@Test func clearTimeoutCancelsAPendingCallback() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const fired = []; + const id = setTimeout(() => { fired.push('should-not-run'); }, 5); + clearTimeout(id); + await new Promise(resolve => setTimeout(resolve, 20)); + return { fired }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output?.objectValue?.array("fired") == []) +} + +@Test func timersFireInDueOrderNotRegistrationOrder() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Registered late-first, so registration order and due order disagree. This + // must hold however far the host's sleep overshoots — if enough time passes + // for both to come due in one tick, they still fire shortest-delay first. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => order.push('b'), 10); + setTimeout(() => order.push('a'), 5); + await new Promise(resolve => setTimeout(resolve, 30)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output?.objectValue?.array("order") == [.string("a"), .string("b")]) +} + +@Test func timersComingDueInOneTickStillFireInDueOrder() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Delays below the host's sleep granularity, so both timers reliably come due + // inside a *single* advanceTimers call — the case that sorting by + // registration id alone got wrong. Registered late-first so the two orderings + // disagree; this fails deterministically against the old implementation + // rather than only on a loaded runner. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => order.push('later'), 2); + setTimeout(() => order.push('sooner'), 1); + await new Promise(resolve => setTimeout(resolve, 40)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output?.objectValue?.array("order") == [.string("sooner"), .string("later")]) +} + +@Test func timersDueAtTheSameInstantFireInRegistrationOrder() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Equal delays, so only registration order can break the tie. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => order.push('first'), 5); + setTimeout(() => order.push('second'), 5); + setTimeout(() => order.push('third'), 5); + await new Promise(resolve => setTimeout(resolve, 30)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect( + observed.result?.output?.objectValue?.array("order") + == [.string("first"), .string("second"), .string("third")] + ) +} + +@Test func anErrorInATimerCallbackBecomesADiagnosticNotACallerThrow() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + setTimeout(() => { throw new Error('boom'); }, 1); + await new Promise(resolve => setTimeout(resolve, 20)); + return 'survived'; + """, + allowedCapabilities: [] + ) + ) + + // Previously this propagated to whoever called setTimeout, which is not how + // an event loop behaves. + #expect(observed.result?.output == .string("survived")) + let diagnostics = try #require(observed.result?.diagnostics) + #expect(diagnostics.contains { $0.code == "TIMER_CALLBACK_ERROR" && $0.message.contains("boom") }) +} + +@Test func anUnsettleablePromiseIsReportedImmediately() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // No resolve path and no queued timer: nothing in this runtime can advance + // it, so sleeping a thread to the deadline buys nothing. + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(() => {}); return 1;", + allowedCapabilities: [], + timeoutMs: 30_000 + ) + ) + + #expect(observed.error?.code == "JS_RUNTIME_ERROR") + #expect(observed.error?.message.contains("can never settle") == true) + // Proves the 30s budget was not waited out. Loose against runner load; the + // margin against 30s is what makes it meaningful. + #expect(Date().timeIntervalSince(started) < 20) +} + +@Test func aTimerStillPendingAtTheDeadlineTimesOut() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // A timer due long after the deadline must not be waited out. + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 30000)); return 1;", + allowedCapabilities: [], + timeoutMs: 100 + ) + ) + + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // The claim is that the 30s timer was not waited out, not that the run was + // fast. Loose enough to survive a loaded runner, tight enough to still catch + // the regression it exists for. + #expect(Date().timeIntervalSince(started) < 20) +} + +@Test func aLongBackoffStaysCancellable() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 20000)); return 1;", + allowedCapabilities: [], + timeoutMs: 60_000 + ) + ) + + Task { + try? await Task.sleep(nanoseconds: 100_000_000) + call.cancel() + } + + let started = Date() + let observed = await observe(call) + #expect(observed.error?.code == "CANCELLED") + // Proves cancel interrupted the 20s timer rather than waiting it out. + #expect(Date().timeIntervalSince(started) < 15) +} + +// MARK: - Swallowed bridge failures + +@Test func aForgottenAwaitOnAFailedCallIsSurfacedAsADiagnostic() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Nothing installs an unhandled-rejection hook, so this used to complete with + // the agent told "done" while a CAPABILITY_DENIED was silently discarded. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + apple.keychain.get({ key: 'never-allowed' }).catch(() => {}); + await new Promise(resolve => setTimeout(resolve, 10)); + return 'looks fine'; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output == .string("looks fine")) + let diagnostics = try #require(observed.result?.diagnostics) + let swallowed = try #require(diagnostics.first { $0.code == "BRIDGE_FAILURES_NOT_SURFACED" }) + #expect(swallowed.message.contains("keychain.read")) + #expect(swallowed.message.contains("CAPABILITY_DENIED")) + #expect(swallowed.suggestions.contains { $0.contains("missing await") }) +} + +@Test func aCleanRunGetsNoSwallowedFailureDiagnostic() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await apple.fs.write({ path: 'tmp:ok.txt', data: 'x' }); return 'ok';", + allowedCapabilities: [.fsWrite] + ) + ) + + #expect(observed.result?.diagnostics.contains { $0.code == "BRIDGE_FAILURES_NOT_SURFACED" } == false) +} + +// MARK: - Bounded concurrency + +@Test func concurrentExecutionsAllCompleteAndStayIsolated() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 3) + ) + defer { cleanup(sandbox) } + + // The concurrent execution queue is a core design point but had no test, and + // no width limit: each execution blocks its thread for the whole run, so N + // parallel calls meant N blocked GCD threads and N live JavaScriptCore VMs. + // More executions than slots must still all complete, each with its own + // context and its own result. + let results = try await withThrowingTaskGroup(of: (Int, JSONValue?).self) { group in + for index in 0..<12 { + group.addTask { + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.fs.write({ path: 'tmp:worker-\(index).txt', data: 'value-\(index)' }); + const read = await apple.fs.read({ path: 'tmp:worker-\(index).txt' }); + return { index: \(index), text: read.text, leaked: typeof globalThis.__leaked }; + """, + allowedCapabilities: [.fsRead, .fsWrite], + timeoutMs: 20_000 + ) + ) + return (index, observed.result?.output) + } + } + + var collected: [Int: JSONValue?] = [:] + for try await (index, output) in group { + collected[index] = output + } + return collected + } + + #expect(results.count == 12) + for index in 0..<12 { + let output = try #require(results[index]??.objectValue) + #expect(output.int("index") == index) + #expect(output.string("text") == "value-\(index)") + // A fresh JSContext per execution: no state carries between them. + #expect(output.string("leaked") == "undefined") + } +} + +@Test func executionsBeyondTheSlotLimitQueueRatherThanFail() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 1) + ) + defer { cleanup(sandbox) } + + // With one slot these serialize; the point is that the extras wait for a slot + // instead of erroring, and that waiting for a slot does not count against the + // per-execution timeout. + let outputs = try await withThrowingTaskGroup(of: JSONValue?.self) { group in + for index in 0..<4 { + group.addTask { + try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(r => setTimeout(r, 60)); return \(index);", + allowedCapabilities: [], + timeoutMs: 500 + ) + ).result?.output + } + } + var collected: [JSONValue?] = [] + for try await output in group { + collected.append(output) + } + return collected + } + + #expect(outputs.compactMap { $0?.intValue }.sorted() == [0, 1, 2, 3]) +} + +@Test func aResultThatSettlesRightAtTheDeadlineIsNotDiscarded() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // The settled state must be read before the deadline is enforced. Checking + // the deadline first turns a script that fulfilled a hair late into a + // spurious timeout, discarding a result that already exists. + for _ in 0..<12 { + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 40)); return 'settled';", + allowedCapabilities: [], + timeoutMs: 40 + ) + ) + // Either outcome is legitimate under the race, but a fulfilled script + // must never come back empty *and* successful. + if observed.error == nil { + #expect(observed.result?.output == .string("settled")) + } else { + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + } + } +} diff --git a/Tests/CodeModeTests/ExecutionLimitsTests.swift b/Tests/CodeModeTests/ExecutionLimitsTests.swift new file mode 100644 index 0000000..7dae8ec --- /dev/null +++ b/Tests/CodeModeTests/ExecutionLimitsTests.swift @@ -0,0 +1,131 @@ +import Foundation +import Testing +@testable import CodeMode + +// Nothing bounded a script's output before: the whole result was stringified and +// copied into Swift, every console.log was retained and re-copied into each +// CodeModeToolError, and the events stream buffered without limit. One script +// could jetsam an iOS host. Every limit degrades with a diagnostic instead of +// failing, because for an LLM consumer a truncated result beats an OOM. + +@Test func oversizedResultsAreReplacedWithAParseableDescriptor() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxResultCharacters: 2_000, truncatedResultPreviewCharacters: 200) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "return Array.from({ length: 5000 }, (_, i) => 'entry-' + i);", + allowedCapabilities: [] + ) + ) + + #expect(observed.error == nil) + let output = try #require(observed.result?.output?.objectValue) + #expect(output.bool("__codeModeTruncated") == true) + #expect((output.int("characterCount") ?? 0) > 2_000) + #expect(output.int("limit") == 2_000) + #expect((output.string("preview")?.count ?? 0) <= 200) + + let diagnostics = try #require(observed.result?.diagnostics) + #expect(diagnostics.contains { $0.code == "RESULT_TRUNCATED" }) +} + +@Test func resultsWithinTheLimitAreUntouched() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxResultCharacters: 100_000) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest(code: "return { ok: true, items: [1, 2, 3] };", allowedCapabilities: []) + ) + + let output = try #require(observed.result?.output?.objectValue) + #expect(output.bool("ok") == true) + #expect(output.array("items")?.count == 3) + #expect(observed.result?.diagnostics.contains { $0.code == "RESULT_TRUNCATED" } == false) +} + +@Test func retainedLogsAreCappedWithANotice() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxLogEntries: 25) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "for (let i = 0; i < 500; i++) { console.log('line ' + i); } return 'done';", + allowedCapabilities: [] + ) + ) + + let logs = try #require(observed.result?.logs) + #expect(logs.count == 25) + // The first N are kept: the beginning of a runaway log is where the cause is. + #expect(logs.first?.message == "line 0") + #expect(observed.result?.diagnostics.contains { $0.code == "LOG_LIMIT_REACHED" } == true) +} + +@Test func longLogMessagesAreElidedInTheMiddle() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxLogMessageCharacters: 200) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "console.log('HEAD' + 'x'.repeat(100000) + 'TAIL'); return 'done';", + allowedCapabilities: [] + ) + ) + + let message = try #require(observed.result?.logs.first?.message) + #expect(message.count < 400) + #expect(message.hasPrefix("HEAD")) + #expect(message.hasSuffix("TAIL")) + #expect(message.contains("characters elided")) +} + +@Test func diagnosticsAndPermissionEventsAreCapped() { + let transcript = ExecutionTranscript( + limits: ExecutionLimits(maxDiagnostics: 3, maxPermissionEvents: 2) + ) + + for index in 0..<50 { + transcript.record(diagnostic: ToolDiagnostic(severity: .info, code: "D\(index)", message: "m")) + transcript.record(permissionEvent: PermissionEvent(permission: .contacts, status: .granted)) + } + + let snapshot = transcript.snapshot() + #expect(snapshot.diagnostics.count == 3) + #expect(snapshot.permissionEvents.count == 2) +} + +@Test func theEventStreamBufferIsBounded() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxLogEntries: 100_000, maxBufferedEvents: 16) + ) + defer { cleanup(sandbox) } + + // Never drained until the script has finished: with the default unbounded + // policy this buffers every event; bounded, it keeps only the newest. + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "for (let i = 0; i < 5000; i++) { console.log('line ' + i); } return 'done';", + allowedCapabilities: [] + ) + ) + _ = try await call.result + + var received = 0 + for await _ in call.events { + received += 1 + } + #expect(received <= 16) +} diff --git a/Tests/CodeModeTests/ExecutionWatchdogTests.swift b/Tests/CodeModeTests/ExecutionWatchdogTests.swift index 1e6012d..3c32f14 100644 --- a/Tests/CodeModeTests/ExecutionWatchdogTests.swift +++ b/Tests/CodeModeTests/ExecutionWatchdogTests.swift @@ -18,7 +18,10 @@ import Testing #expect(observed.result == nil) #expect(observed.error?.code == "EXECUTION_TIMEOUT") - #expect(Date().timeIntervalSince(started) < 5) + // A hang-guard, not a performance assertion: `while (true) {}` must be + // terminated at all. The error code above is the real check, and the bound is + // loose because a shared CI runner is not an idle machine. + #expect(Date().timeIntervalSince(started) < 20) } @Test func executeTerminatesCPUBoundLoopInsidePromiseChain() async throws { @@ -151,8 +154,34 @@ import Testing ) #expect(observed.result?.output == nil) - #expect(observed.error != nil) - #expect(Date().timeIntervalSince(started) < 5) + // The watchdog fired, so this is a timeout — not the INVALID_RESULT + // "must be JSON-serializable" it used to report, which pointed the model at + // the wrong repair entirely. + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // Same hang-guard reasoning: the runaway getter must be terminated at all. + #expect(Date().timeIntervalSince(started) < 20) +} + +@Test func serializationBudgetDoesNotScaleWithTheExecutionTimeout() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // A long execution budget used to buy an equally long serialization budget + // (`max`, not `min`), so a runaway getter after a 30s-budget script got + // another 30s. The serialization phase is bounded independently. + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "return { get trap() { while (true) {} } };", + allowedCapabilities: [], + timeoutMs: 30_000 + ) + ) + + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // Proves serialization did not inherit the script's 30s budget. + #expect(Date().timeIntervalSince(started) < 20) } @Test func searchTerminatesCPUBoundInfiniteLoop() async throws { diff --git a/Tests/CodeModeTests/FileSystemBridgeTests.swift b/Tests/CodeModeTests/FileSystemBridgeTests.swift index 2cae75c..f7482ac 100644 --- a/Tests/CodeModeTests/FileSystemBridgeTests.swift +++ b/Tests/CodeModeTests/FileSystemBridgeTests.swift @@ -239,3 +239,188 @@ private final class RecordingCodeModeFileSystem: CodeModeFileSystem, @unchecked let payload = try requireJSONObject(from: try #require(observed.result)) #expect(payload["text"] as? String == "fs-from-execute") } + +// MARK: - Destructive-destination guards + +@Test func fileSystemMoveRefusesASandboxRootAsDestination() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:junk.txt"), "data": .string("x")], context: context) + _ = try fs.write(arguments: ["path": .string("documents:keepme.txt"), "data": .string("precious")], context: context) + + // `documents:` resolves to the documents root itself, and containment admits + // it. The old unconditional removeItem here deleted the user's Documents. + for destination in ["documents:", "documents:/", "tmp:", "caches:"] { + do { + _ = try fs.move(arguments: [ + "from": .string("tmp:junk.txt"), + "to": .string(destination), + "overwrite": .bool(true), + "recursive": .bool(true), + ], context: context) + Issue.record("Expected fs.move to refuse the root destination \(destination)") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } + } + + let survived = try fs.read(arguments: ["path": .string("documents:keepme.txt")], context: context) + #expect(try requireObject(survived).string("text") == "precious") +} + +@Test func fileSystemCopyRefusesASandboxRootAsDestination() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:junk.txt"), "data": .string("x")], context: context) + + #expect(throws: (any Error).self) { + _ = try fs.copy(arguments: [ + "from": .string("tmp:junk.txt"), + "to": .string("documents:"), + "overwrite": .bool(true), + ], context: context) + } +} + +@Test func fileSystemDeleteRefusesASandboxRoot() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + do { + _ = try fs.delete(arguments: ["path": .string("documents:"), "recursive": .bool(true)], context: context) + Issue.record("Expected fs.delete to refuse a sandbox root") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } + #expect(FileManager.default.fileExists(atPath: sandbox.documents.path)) +} + +@Test func fileSystemMoveAndCopyRequireExplicitOverwrite() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:a.txt"), "data": .string("source")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:b.txt"), "data": .string("destination")], context: context) + + do { + _ = try fs.copy(arguments: ["from": .string("tmp:a.txt"), "to": .string("tmp:b.txt")], context: context) + Issue.record("Expected fs.copy to refuse an existing destination without overwrite") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + + // The destination is untouched by the refused call. + let intact = try fs.read(arguments: ["path": .string("tmp:b.txt")], context: context) + #expect(try requireObject(intact).string("text") == "destination") + + _ = try fs.copy(arguments: [ + "from": .string("tmp:a.txt"), + "to": .string("tmp:b.txt"), + "overwrite": .bool(true), + ], context: context) + let replaced = try fs.read(arguments: ["path": .string("tmp:b.txt")], context: context) + #expect(try requireObject(replaced).string("text") == "source") +} + +@Test func fileSystemMoveOntoADirectoryAlsoRequiresRecursive() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:a.txt"), "data": .string("source")], context: context) + _ = try fs.mkdir(arguments: ["path": .string("tmp:dir")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:dir/kept.txt"), "data": .string("kept")], context: context) + + do { + _ = try fs.move(arguments: [ + "from": .string("tmp:a.txt"), + "to": .string("tmp:dir"), + "overwrite": .bool(true), + ], context: context) + Issue.record("Expected fs.move onto a directory to require recursive=true") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + #expect(try fs.exists(arguments: ["path": .string("tmp:dir/kept.txt")], context: context).boolValue == true) + + _ = try fs.move(arguments: [ + "from": .string("tmp:a.txt"), + "to": .string("tmp:dir"), + "overwrite": .bool(true), + "recursive": .bool(true), + ], context: context) + #expect(try fs.exists(arguments: ["path": .string("tmp:dir/kept.txt")], context: context).boolValue == false) +} + +// MARK: - Size and decoding limits + +@Test func fileSystemReadRefusesFilesOverTheLimit() throws { + let fs = FileSystemBridge(limits: FileSystemLimits(maxReadBytes: 64, maxWriteBytes: 1_024)) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: [ + "path": .string("tmp:big.txt"), + "data": .string(String(repeating: "x", count: 512)), + ], context: context) + + do { + _ = try fs.read(arguments: ["path": .string("tmp:big.txt")], context: context) + Issue.record("Expected fs.read to refuse a file over the read limit") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } +} + +@Test func fileSystemWriteRefusesPayloadsOverTheLimit() throws { + let fs = FileSystemBridge(limits: FileSystemLimits(maxReadBytes: 1_024, maxWriteBytes: 16)) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + do { + _ = try fs.write(arguments: [ + "path": .string("tmp:nested/big.txt"), + "data": .string(String(repeating: "x", count: 512)), + ], context: context) + Issue.record("Expected fs.write to refuse a payload over the write limit") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + + // A refused write leaves no directory behind. + #expect(try fs.exists(arguments: ["path": .string("tmp:nested")], context: context).boolValue == false) +} + +@Test func fileSystemReadReportsUndecodableUTF8InsteadOfEmptyText() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + // 0xFF is never valid UTF-8. Returning "" here told the script the file was + // empty; it must be an error the script can repair with base64. + let binary = Data([0xFF, 0xFE, 0x00, 0x01]) + _ = try fs.write(arguments: [ + "path": .string("tmp:binary.bin"), + "data": .string(binary.base64EncodedString()), + "encoding": .string("base64"), + ], context: context) + + do { + _ = try fs.read(arguments: ["path": .string("tmp:binary.bin")], context: context) + Issue.record("Expected fs.read to fail on undecodable UTF-8") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + + let base64 = try fs.read(arguments: [ + "path": .string("tmp:binary.bin"), + "encoding": .string("base64"), + ], context: context) + #expect(try requireObject(base64).string("base64") == binary.base64EncodedString()) +} diff --git a/Tests/CodeModeTests/HostRuntimeTests.swift b/Tests/CodeModeTests/HostRuntimeTests.swift index 158798a..b44c8b4 100644 --- a/Tests/CodeModeTests/HostRuntimeTests.swift +++ b/Tests/CodeModeTests/HostRuntimeTests.swift @@ -602,7 +602,11 @@ import Testing ) #expect(observed.result == nil) - #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // A promise with no resolve path and no pending timer is provably + // unsettleable under this runtime, so it is reported immediately instead of + // sleeping a thread until the deadline for a result that can never arrive. + #expect(observed.error?.code == "JS_RUNTIME_ERROR") + #expect(observed.error?.message.contains("can never settle") == true) } @Test func executeRecoversWithFreshContextAfterTimeout() async throws { @@ -617,7 +621,7 @@ import Testing timeoutMs: 30 ) ) - #expect(timedOut.error?.code == "EXECUTION_TIMEOUT") + #expect(timedOut.error?.code == "JS_RUNTIME_ERROR") let recovered = try await execute( tools, @@ -700,13 +704,17 @@ import Testing let (tools, sandbox) = try makeTools() defer { cleanup(sandbox) } + // A long timer, not an unsettleable promise: the latter now fails fast with + // JS_RUNTIME_ERROR, which would win the race against the cancellation this + // test is about. let call = try await tools.executeJavaScript( JavaScriptExecutionRequest( code: """ - await new Promise(() => {}); + await new Promise(resolve => setTimeout(resolve, 20000)); return { never: true }; """, - allowedCapabilities: [] + allowedCapabilities: [], + timeoutMs: 60_000 ) ) diff --git a/Tests/CodeModeTests/NetworkBridgeTests.swift b/Tests/CodeModeTests/NetworkBridgeTests.swift index 5e444ae..ec72f4e 100644 --- a/Tests/CodeModeTests/NetworkBridgeTests.swift +++ b/Tests/CodeModeTests/NetworkBridgeTests.swift @@ -227,3 +227,74 @@ private final class StubHTTPURLProtocol: URLProtocol { #expect(result.string("stringValue") == "https://example.com/path?q=1") #expect(result.string("methodValue") == "https://example.com/path?q=1") } + +// MARK: - Ambient authority + +@Test func networkBridgeDefaultSessionCarriesNoAmbientCredentials() { + // The shipping default used to be `URLSession.shared`, which reads + // HTTPCookieStorage.shared and URLCredentialStorage.shared — so a script got + // authenticated session-riding against every origin the app is logged in to, + // and a Set-Cookie in a script-fetched response poisoned the app's jar. + let configuration = NetworkBridge.isolatedSession.configuration + #expect(configuration.httpCookieStorage == nil) + #expect(configuration.urlCredentialStorage == nil) + #expect(configuration.urlCache == nil) + #expect(configuration.httpShouldSetCookies == false) + #expect(configuration.httpCookieAcceptPolicy == .never) + #expect(NetworkBridge.isolatedSession !== URLSession.shared) +} + +@Test func networkFetchRefusesCredentialHeadersByDefault() throws { + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [StubHTTPURLProtocol.self] + let session = URLSession(configuration: configuration) + + let bridge = NetworkBridge(session: session) + let (context, sandbox) = try makeInvocationContext() + defer { + cleanup(sandbox) + session.invalidateAndCancel() + } + + for header in ["Cookie", "cookie", "Authorization", "Proxy-Authorization", "proxy-authenticate"] { + do { + _ = try bridge.fetch(arguments: [ + "url": .string("https://unit.test/endpoint"), + "options": .object(["headers": .object([header: .string("secret")])]), + ], context: context) + Issue.record("Expected \(header) to be refused") + } catch { + #expect(requireBridgeErrorCode(error) == "NETWORK_POLICY_VIOLATION") + } + } + + // Ordinary headers are unaffected. + let ok = try bridge.fetch(arguments: [ + "url": .string("https://unit.test/endpoint"), + "options": .object(["headers": .object(["X-Unit": .string("plain")])]), + ], context: context) + #expect(try requireObject(ok).bool("ok") == true) +} + +@Test func networkFetchAllowsCredentialHeadersWhenHostOptsIn() throws { + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [StubHTTPURLProtocol.self] + let session = URLSession(configuration: configuration) + + let bridge = NetworkBridge( + session: session, + policy: NetworkAccessPolicy(allowsCredentialHeaders: true) + ) + let (context, sandbox) = try makeInvocationContext() + defer { + cleanup(sandbox) + session.invalidateAndCancel() + } + + let result = try bridge.fetch(arguments: [ + "url": .string("https://unit.test/endpoint"), + "options": .object(["headers": .object(["Authorization": .string("Bearer t")])]), + ], context: context) + #expect(try requireObject(result).bool("ok") == true) + #expect(NetworkAccessPolicy.permissive.allowsCredentialHeaders) +} diff --git a/Tests/CodeModeTests/PolishFindingsTests.swift b/Tests/CodeModeTests/PolishFindingsTests.swift new file mode 100644 index 0000000..ac3f7cb --- /dev/null +++ b/Tests/CodeModeTests/PolishFindingsTests.swift @@ -0,0 +1,143 @@ +import Foundation +import Testing +@testable import CodeMode + +// MARK: - One ISO 8601 parser + +@Test func iso8601ParsingAcceptsFractionalSeconds() { + // `Date.toISOString()` in JavaScript always emits milliseconds, so this is + // the spelling a model produces by default. A bare ISO8601DateFormatter + // rejects it, and the bridges that used one silently substituted "now". + #expect(CodeModeDate.parse("2026-07-24T10:00:00.000Z") != nil) + #expect(CodeModeDate.parse("2026-07-24T10:00:00Z") != nil) + #expect(CodeModeDate.parse("2026-07-24T10:00:00+01:00") != nil) + #expect(CodeModeDate.parse(" 2026-07-24T10:00:00Z ") != nil) + #expect(CodeModeDate.parse("not a date") == nil) + #expect(CodeModeDate.parse("") == nil) + #expect(CodeModeDate.parse(nil) == nil) +} + +@Test func requiredDatesFailInsteadOfSubstitutingADefault() { + #expect(throws: (any Error).self) { + _ = try CodeModeDate.require("yesterday", argument: "start", capability: "calendar.read") + } + #expect(throws: (any Error).self) { + _ = try CodeModeDate.require(nil, argument: "start", capability: "calendar.read") + } + // A present-but-unparseable optional is still an error; only absence defaults. + #expect(throws: (any Error).self) { + _ = try CodeModeDate.optional("soon", argument: "end", capability: "calendar.read") + } +} + +@Test func absentOptionalDatesStillFallBackToTheCallersDefault() throws { + let absent = try CodeModeDate.optional(nil, argument: "end", capability: "calendar.read") + #expect(absent == nil) +} + +// MARK: - Network policy + +@Test func standardPolicyBlocksBareSingleLabelHosts() { + let policy = NetworkAccessPolicy.standard + // These resolve onto the LAN through the DHCP search domain, exactly like + // *.local, but previously fell through to "allowed". + for host in ["router", "intranet", "wpad", "nas", "printer"] { + let url = URL(string: "http://\(host)/admin")! + #expect(policy.violationReason(for: url) != nil, "expected \(host) to be refused") + } + + // Ordinary public hosts are unaffected. + #expect(policy.violationReason(for: URL(string: "https://example.com/x")!) == nil) + #expect(policy.violationReason(for: URL(string: "https://api.example.co.uk/x")!) == nil) +} + +@Test func hostsCanStillAllowlistASingleLabelHost() { + let policy = NetworkAccessPolicy(allowedHosts: ["router"]) + #expect(policy.violationReason(for: URL(string: "http://router/admin")!) == nil) +} + +// MARK: - System UI destinations + +@Test func webPresentationRespectsTheNetworkAccessPolicy() throws { + let bridge = SystemUIBridge(networkAccessPolicy: .standard) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + // Opening a page in a browser was exempt from the policy governing fetch, so + // a blocked origin could still be put in front of the user — or be the target + // of a script-started OAuth flow. + for capability in ["web", "auth"] { + let arguments: [String: JSONValue] = ["url": .string("http://169.254.169.254/latest/meta-data")] + do { + if capability == "web" { + _ = try bridge.presentWeb(arguments: arguments, context: context) + } else { + _ = try bridge.authenticateWeb(arguments: arguments, context: context) + } + Issue.record("Expected \(capability) presentation to be refused by the network policy") + } catch { + #expect(requireBridgeErrorCode(error) == "NETWORK_POLICY_VIOLATION") + } + } +} + +// MARK: - Path containment + +@Test func danglingSymlinksDoNotPassContainment() throws { + let sandbox = try makeTestSandbox() + defer { cleanup(sandbox) } + + let outsideRoot = sandbox.root.appendingPathComponent("outside", isDirectory: true) + let policy = DefaultPathPolicy( + config: PathPolicyConfig(tmpRoot: sandbox.tmp, cachesRoot: sandbox.caches, documentsRoot: sandbox.documents) + ) + + // The target deliberately does not exist: `fileExists` follows symlinks and + // so reported the link as a *missing component*, which was re-appended to the + // resolved ancestor and admitted instead of being resolved out of the root. + let link = sandbox.tmp.appendingPathComponent("escape") + try FileManager.default.createSymbolicLink( + at: link, + withDestinationURL: outsideRoot.appendingPathComponent("secret.txt") + ) + + #expect(throws: (any Error).self) { + _ = try policy.resolve(path: "tmp:escape") + } +} + +@Test func allowedRootsAreReportedByTheDefaultPolicy() throws { + let sandbox = try makeTestSandbox() + defer { cleanup(sandbox) } + + let policy = DefaultPathPolicy( + config: PathPolicyConfig(tmpRoot: sandbox.tmp, cachesRoot: sandbox.caches, documentsRoot: sandbox.documents) + ) + let documentsRoot = try policy.resolve(path: "documents:") + let tmpRoot = try policy.resolve(path: "tmp:") + let insideTmp = try policy.resolve(path: "tmp:file.txt") + + #expect(policy.allowedRoots.count == 3) + #expect(policy.isAllowedRoot(documentsRoot)) + #expect(policy.isAllowedRoot(tmpRoot)) + #expect(policy.isAllowedRoot(insideTmp) == false) +} + +// MARK: - Diagnostics reach the stream + +@Test func errorSeverityDiagnosticsAreEmittedToTheEventStream() { + let received = LockedBox<[ToolDiagnostic]>([]) + let transcript = ExecutionTranscript { event in + if case let .diagnostic(diagnostic) = event { + received.set(received.get() + [diagnostic]) + } + } + + transcript.record(diagnostic: ToolDiagnostic(severity: .info, code: "I", message: "info")) + transcript.record(diagnostic: ToolDiagnostic(severity: .warning, code: "W", message: "warning")) + // Previously filtered out, so a host watching the stream saw every diagnostic + // except the ones that mattered most. + transcript.record(diagnostic: ToolDiagnostic(severity: .error, code: "E", message: "error")) + + #expect(received.get().map(\.code) == ["I", "W", "E"]) +} diff --git a/Tests/CodeModeTests/TestSupport.swift b/Tests/CodeModeTests/TestSupport.swift index 3e2a23a..6973d2b 100644 --- a/Tests/CodeModeTests/TestSupport.swift +++ b/Tests/CodeModeTests/TestSupport.swift @@ -35,6 +35,8 @@ func cleanup(_ sandbox: TestSandbox) { func makeTools( permissionBroker: any PermissionBroker = NoopPermissionBroker(), + capabilityGrant: CapabilityGrant = .unrestricted, + executionLimits: ExecutionLimits = .standard, fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), systemUIPresenter: any SystemUIPresenter = UnavailableSystemUIPresenter(), eventInbox: any CodeModeEventInbox = UnavailableCodeModeEventInbox(), @@ -59,7 +61,9 @@ func makeTools( let configuration = CodeModeConfiguration( pathPolicy: pathPolicy, + capabilityGrant: capabilityGrant, fileSystem: fileSystem, + executionLimits: executionLimits, artifactStore: InMemoryArtifactStore(), permissionBroker: permissionBroker, auditLogger: SyncAuditLogger(), diff --git a/Tests/CodeModeTests/TypeScriptDeclarationsTests.swift b/Tests/CodeModeTests/TypeScriptDeclarationsTests.swift new file mode 100644 index 0000000..4421ea2 --- /dev/null +++ b/Tests/CodeModeTests/TypeScriptDeclarationsTests.swift @@ -0,0 +1,199 @@ +import Foundation +import Testing +@testable import CodeMode + +// Cloudflare's central Code Mode insight is that models write markedly better +// code against real types. Before this the model got a flat `argumentTypes` map +// of six coarse cases plus a prose `resultSummary`; these tests pin the shape of +// what it gets now. + +private func reference( + capability: String = "fs.read", + jsNames: [String] = ["apple.fs.read", "fs.promises.readFile"], + required: [String] = ["path"], + optional: [String] = ["encoding"], + types: [String: CapabilityArgumentType] = ["path": .string, "encoding": .string], + hints: [String: String] = ["path": "File path using allowed root prefix."], + constraints: CapabilityArgumentConstraints = .init(allowedStringValues: ["encoding": ["utf8", "base64"]]) +) -> JavaScriptAPIReference { + JavaScriptAPIReference( + capability: capability, + jsNames: jsNames, + summary: "Read a file.", + tags: ["fs"], + example: "await apple.fs.read({ path: 'tmp:x.txt' })", + requiredArguments: required, + optionalArguments: optional, + argumentTypes: types, + argumentHints: hints, + argumentConstraints: constraints, + resultSummary: "Object with path plus text or base64." + ) +} + +@Test func declarationsTypeArgumentsAndCarryHintsAsDocComments() { + let dts = TypeScriptDeclarations.declaration(for: reference()) + + #expect(dts.contains("path: string;")) + #expect(dts.contains("/** File path using allowed root prefix. */")) + // Required vs optional is expressed in the type, not left to prose. + #expect(dts.contains("encoding?:")) + #expect(dts.contains("path?:") == false) + #expect(dts.contains("Capability: fs.read")) + #expect(dts.contains("Returns: Object with path plus text or base64.")) +} + +@Test func constrainedArgumentsBecomeStringLiteralUnions() { + let dts = TypeScriptDeclarations.declaration(for: reference()) + // The constraint IS the type; a union beats `string` plus a prose hint the + // model has to notice and obey. + #expect(dts.contains(#"encoding?: "utf8" | "base64";"#)) +} + +@Test func nodeStyleAliasesAreDocumentedRatherThanMistypedAsObjectCalls() { + let dts = TypeScriptDeclarations.declaration(for: reference()) + // `fs.promises.readFile` is positional, which the catalog cannot express, so + // the generated signature must describe the namespaced helper and only + // mention the alias. + #expect(dts.contains("Aliases: fs.promises.readFile")) + #expect(dts.contains("positional")) +} + +@Test func dottedArgumentPathsBecomeNestedObjectTypes() { + let nested = reference( + capability: "network.fetch", + jsNames: ["fetch"], + required: ["url"], + optional: ["options.method", "options.timeoutMs"], + types: ["url": .string, "options.method": .string, "options.timeoutMs": .number], + hints: ["options.method": "HTTP method; defaults to GET."], + constraints: .none + ) + let dts = TypeScriptDeclarations.declaration(for: nested) + + #expect(dts.contains("options?: {")) + #expect(dts.contains("method?: string;")) + #expect(dts.contains("timeoutMs?: number;")) +} + +@Test func argumentlessHelpersTakeNoArgument() { + let none = reference( + capability: "location.permission.request", + jsNames: ["apple.location.requestPermission"], + required: [], + optional: [], + types: [:], + hints: [:], + constraints: .none + ) + #expect(TypeScriptDeclarations.declaration(for: none).contains("(): Promise")) +} + +@Test func theWholeSurfaceIsNamespacedAndPlatformFiltered() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .macOS) + defer { cleanup(sandbox) } + + let surface = tools.typeDeclarations() + + #expect(surface.contains("declare namespace apple {")) + #expect(surface.contains("namespace fs {")) + #expect(surface.contains("function read(args: {")) + // The Node-compat globals live in the hand-authored preamble, because their + // positional convention is not something the catalog can describe. + #expect(surface.contains("declare function fetch(")) + #expect(surface.contains("type CodeModeValue =")) + // iOS-only helpers must not be declared for a macOS host. + #expect(surface.contains("namespace alarm {") == false) +} + +@Test func iOSOnlyHelpersAppearForAniOSHost() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + #expect(tools.typeDeclarations().contains("namespace alarm {")) +} + +@Test func everyCapabilityCarriesItsOwnDeclarationForSearchResults() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let references = tools.capabilities() + #expect(references.isEmpty == false) + #expect(references.allSatisfy { $0.dts.isEmpty == false }) + + let read = try #require(references.first { $0.capability == "fs.read" }) + #expect(read.dts.contains("path: string;")) +} + +@Test func searchCanReturnDeclarationsAsItsPayload() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Code-driven search stays the filter; the payload is now TypeScript. + let response = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest( + code: """ + async () => { + return api.references + .filter(ref => ref.tags.includes("filesystem")) + .map(ref => ref.dts) + .join("\\n\\n"); + } + """ + ) + ) + let payload = try #require(response.result?.stringValue) + #expect(payload.contains("function apple_fs_read(args: {")) + #expect(payload.contains("path: string;")) +} + +@Test func nodeAliasesCanPassTheOverwriteFlagTheDeclarationPromises() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // fs.move/fs.copy now refuse an existing destination without an explicit + // overwrite, so the Node-style aliases had to gain a way to pass one — the + // preamble declares it, and this proves the runtime honours it. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.fs.write({ path: 'tmp:src.txt', data: 'source' }); + await apple.fs.write({ path: 'tmp:dst.txt', data: 'destination' }); + await fs.promises.copyFile('tmp:src.txt', 'tmp:dst.txt', { overwrite: true }); + return await fs.promises.readFile('tmp:dst.txt', 'utf8'); + """, + allowedCapabilities: [.fsWrite, .fsRead, .fsCopy] + ) + ) + + #expect(observed.error == nil) + #expect(observed.result?.output == .string("source")) +} + +@Test func keychainHelpersAcceptTheObjectFormTheCatalogAdvertises() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // The catalog declared `key: string` as an object argument while the wrapper + // took a positional string, and its own example showed the positional form — + // so code written from the metadata sent "[object Object]" as the key. Both + // spellings now work, which is what makes the generated declaration true. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.keychain.set({ key: 'codemode-dts-test', value: 'object-form' }); + const viaObject = await apple.keychain.get({ key: 'codemode-dts-test' }); + const viaPositional = await apple.keychain.get('codemode-dts-test'); + await apple.keychain.delete({ key: 'codemode-dts-test' }); + return { viaObject: viaObject && viaObject.value, viaPositional: viaPositional && viaPositional.value }; + """, + allowedCapabilities: [.keychainRead, .keychainWrite, .keychainDelete] + ) + ) + + #expect(observed.error == nil) + let output = try #require(observed.result?.output?.objectValue) + #expect(output.string("viaObject") == "object-form") + #expect(output.string("viaPositional") == "object-form") +} diff --git a/Tests/CodeModeTests/capability-metadata-golden.json b/Tests/CodeModeTests/capability-metadata-golden.json index 8e51716..a48e9ab 100644 --- a/Tests/CodeModeTests/capability-metadata-golden.json +++ b/Tests/CodeModeTests/capability-metadata-golden.json @@ -532,7 +532,7 @@ }, "argumentHints" : { "callbackURLScheme" : "Optional custom URL scheme that completes the session.", - "prefersEphemeralSession" : "Whether to prefer a private browser session; default false.", + "prefersEphemeralSession" : "Whether to use a private browser session; default true. Set false only when the flow must reuse the user's existing browser session.", "timeoutMs" : "Optional timeout for waiting on callback\/cancellation.", "url" : "Absolute HTTP(S) authentication URL." }, @@ -559,7 +559,7 @@ ], "resultSummary" : "Object with action callback\/cancelled and callbackURL when available.", - "summary" : "Start an ASWebAuthenticationSession for OAuth-style browser authentication.", + "summary" : "Start an ASWebAuthenticationSession for OAuth-style browser authentication. Runs in a private browser session by default and the URL must satisfy the host's network access policy.", "tags" : [ "auth", "oauth", @@ -1852,10 +1852,14 @@ }, "argumentHints" : { "from" : "Source sandbox path.", - "to" : "Destination sandbox path." + "overwrite" : "Required as true to replace an existing destination; default false.", + "recursive" : "Required as true alongside overwrite when the existing destination is a directory.", + "to" : "Destination sandbox path. Must name a path inside a root, not the root itself." }, "argumentTypes" : { "from" : "string", + "overwrite" : "bool", + "recursive" : "bool", "to" : "string" }, "example" : "await apple.fs.copy({ from: 'tmp:a.txt', to: 'tmp:b.txt' })", @@ -1865,7 +1869,8 @@ "fs.promises.copyFile" ], "optionalArguments" : [ - + "overwrite", + "recursive" ], "requiredArguments" : [ "from", @@ -1875,7 +1880,7 @@ ], "resultSummary" : "Object with from\/to resolved paths.", - "summary" : "Copy file\/directory within allowed sandbox roots.", + "summary" : "Copy file\/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination.", "tags" : [ "filesystem", "io", @@ -2028,10 +2033,14 @@ }, "argumentHints" : { "from" : "Source sandbox path.", - "to" : "Destination sandbox path." + "overwrite" : "Required as true to replace an existing destination; default false.", + "recursive" : "Required as true alongside overwrite when the existing destination is a directory.", + "to" : "Destination sandbox path. Must name a path inside a root, not the root itself." }, "argumentTypes" : { "from" : "string", + "overwrite" : "bool", + "recursive" : "bool", "to" : "string" }, "example" : "await apple.fs.move({ from: 'tmp:a.txt', to: 'tmp:b.txt' })", @@ -2041,7 +2050,8 @@ "fs.promises.rename" ], "optionalArguments" : [ - + "overwrite", + "recursive" ], "requiredArguments" : [ "from", @@ -2051,7 +2061,7 @@ ], "resultSummary" : "Object with from\/to resolved paths.", - "summary" : "Move file\/directory within allowed sandbox roots.", + "summary" : "Move file\/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination.", "tags" : [ "filesystem", "io", @@ -2061,10 +2071,14 @@ }, { "allowedStringValues" : { - + "encoding" : [ + "utf8", + "base64", + "utf-8" + ] }, "argumentHints" : { - "encoding" : "utf8 (default) or base64.", + "encoding" : "utf8 (default) or base64. Binary files must use base64; a utf8 read of undecodable bytes fails rather than returning empty text.", "path" : "File path using allowed root prefix." }, "argumentTypes" : { @@ -2131,7 +2145,11 @@ }, { "allowedStringValues" : { - + "encoding" : [ + "utf8", + "base64", + "utf-8" + ] }, "argumentHints" : { "data" : "UTF-8 text or base64 string depending on encoding.", @@ -2380,7 +2398,7 @@ "argumentTypes" : { "key" : "string" }, - "example" : "await apple.keychain.delete('auth_token')", + "example" : "await apple.keychain.delete({ key: 'auth_token' })", "id" : "keychain.delete", "jsNames" : [ "apple.keychain.delete" @@ -2413,7 +2431,7 @@ "argumentTypes" : { "key" : "string" }, - "example" : "await apple.keychain.get('auth_token')", + "example" : "await apple.keychain.get({ key: 'auth_token' })", "id" : "keychain.read", "jsNames" : [ "apple.keychain.get" @@ -2448,7 +2466,7 @@ "key" : "string", "value" : "string" }, - "example" : "await apple.keychain.set('auth_token', token)", + "example" : "await apple.keychain.set({ key: 'auth_token', value: token })", "id" : "keychain.write", "jsNames" : [ "apple.keychain.set" @@ -3265,7 +3283,7 @@ "argumentHints" : { "options.body" : "UTF-8 request body string.", "options.bodyBase64" : "Base64-encoded request body. Mutually exclusive with options.body.", - "options.headers" : "Object of header key\/value string pairs.", + "options.headers" : "Object of header key\/value string pairs. Cookie, Authorization, and Proxy-* are refused unless the host app permits credential headers.", "options.method" : "HTTP method; defaults to GET.", "options.responseEncoding" : "text (default) or base64.", "options.timeoutMs" : "Request and bridge wait timeout in milliseconds; defaults to 30000.", @@ -3300,7 +3318,7 @@ ], "resultSummary" : "Object with ok\/status\/statusText\/headers\/bodyText or bodyBase64.", - "summary" : "Perform HTTP(S) requests through URLSession via a fetch-compatible API.", + "summary" : "Perform HTTP(S) requests through an isolated URLSession via a fetch-compatible API. Requests carry no app cookies or stored credentials, and credential headers (Cookie, Authorization, Proxy-*) are refused unless the host app permits them.", "tags" : [ "network", "http",