From 3348636b29cd23bb7b87232867c4cce7fe05b7f9 Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 19:42:49 -0700 Subject: [PATCH 01/18] Make the host, not the model, the capability boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `allowedCapabilities`/`allowedCapabilityKeys` are model-authored tool arguments, and nothing in the package narrowed them, so a host that piped tool JSON straight into `executeJavaScript` — the README Quick Start shape — granted whatever the script declared for itself. Adds `CodeModeConfiguration.capabilityGrant` (`CapabilityGrant`), intersected with the request before the invocation context is built, so the effective set is always requested ∩ granted. Defaults to `.unrestricted` for source compatibility. Withheld identifiers produce a `CAPABILITY_WITHHELD_BY_HOST` diagnostic, and the resulting `CAPABILITY_DENIED` tells the model the denial is not repairable by widening its own allowlist, so it stops escalating. Also closes the aliasing hole: `allowedCapabilityKeys` accepts arbitrary strings and is not validated against `CapabilityID` at decode time, but `invokeBuiltIn` honoured it and `registeredFunction(for: key)` resolved built-ins first — so `{"allowedCapabilities": [], "allowedCapabilityKeys": ["keychain.read"]}` reached the built-in keychain bridge past any host vetting only the typed field. Built-ins are now gated by `allowedCapabilities` alone, the key namespace resolves custom providers only, and a key that spells a built-in is rejected on the provider path as defense in depth. Granting a built-in by its `CapabilityID` is unchanged. Review findings P0 #1 and #2. --- README.md | 33 ++++++++ Sources/CodeMode/API/BridgeModels.swift | 5 ++ .../Host/CodeModeAgentToolDescriptions.swift | 6 +- .../Registry/CapabilityRegistry.swift | 22 ++++-- .../Runtime/BridgeInvocationContext.swift | 19 ++++- Sources/CodeMode/Runtime/BridgeRuntime.swift | 77 +++++++++++++++---- .../CodeMode/Security/CapabilityGrant.swift | 76 ++++++++++++++++++ Sources/CodeModeAuthoring/README.md | 6 ++ .../CodeModeTests/CodeModeProviderTests.swift | 75 +++++++++++++++++- Tests/CodeModeTests/TestSupport.swift | 2 + 10 files changed, 296 insertions(+), 25 deletions(-) create mode 100644 Sources/CodeMode/Security/CapabilityGrant.swift diff --git a/README.md b/README.md index 29bf64f..453d109 100644 --- a/README.md +++ b/README.md @@ -215,11 +215,44 @@ async () => { - `code` - `allowedCapabilities` +- `allowedCapabilityKeys` - `timeoutMs` - `context` It returns a `JavaScriptExecutionCall` immediately. +### Capability allowlists are model-authored; the grant is not + +`allowedCapabilities` and `allowedCapabilityKeys` travel in the advertised tool +schema, so the *model* fills them in. They are a least-privilege declaration and +a useful audit signal, but on their own they are not a sandbox: a host that pipes +tool JSON straight into `executeJavaScript` gives the script whatever the script +asked for. + +`CodeModeConfiguration.capabilityGrant` is the host-owned ceiling. The effective +set is always `requested ∩ granted`: + +```swift +let tools = CodeModeAgentTools( + config: CodeModeConfiguration( + capabilityGrant: .only( + [.fsRead, .fsWrite, .networkFetch], + capabilityKeys: ["myapp.tasks.complete"] + ) + ) +) +``` + +It defaults to `.unrestricted` for source compatibility. Anything the request +declares and the grant withholds fails with `CAPABILITY_DENIED`, is reported to +the model as not repairable by retrying, and is recorded as a +`CAPABILITY_WITHHELD_BY_HOST` diagnostic. + +The two allowlists are strictly disjoint. Built-in `CapabilityID`s are granted +only by `allowedCapabilities`; `allowedCapabilityKeys` accepts arbitrary strings +and reaches custom providers only, so it cannot be used to spell a built-in past +a host that vets the typed field. + Cross-platform privileged helpers are installed under `apple.*`. Platform-specific helpers are installed only where supported, for example `ios.alarm.*` on iOS hosts that support AlarmKit. `apple.location.requestPermission()` is currently exposed only on iOS hosts. Other Apple platforms can expose `apple.location.*` helpers when supported, but the explicit permission-request helper is intentionally hidden outside iOS for now. diff --git a/Sources/CodeMode/API/BridgeModels.swift b/Sources/CodeMode/API/BridgeModels.swift index e922dd9..3906e2f 100644 --- a/Sources/CodeMode/API/BridgeModels.swift +++ b/Sources/CodeMode/API/BridgeModels.swift @@ -3,6 +3,9 @@ import Foundation public struct CodeModeConfiguration: Sendable { public var pathPolicy: any PathPolicy public var networkAccessPolicy: NetworkAccessPolicy + /// Host-owned ceiling on the model-authored `allowedCapabilities` / + /// `allowedCapabilityKeys`. Defaults to `.unrestricted`; see `CapabilityGrant`. + public var capabilityGrant: CapabilityGrant public var fileSystem: any CodeModeFileSystem public var artifactStore: any ArtifactStore public var permissionBroker: any PermissionBroker @@ -25,6 +28,7 @@ public struct CodeModeConfiguration: Sendable { public init( pathPolicy: any PathPolicy = DefaultPathPolicy(), networkAccessPolicy: NetworkAccessPolicy = .standard, + capabilityGrant: CapabilityGrant = .unrestricted, fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), artifactStore: any ArtifactStore = InMemoryArtifactStore(), permissionBroker: any PermissionBroker = SystemPermissionBroker(), @@ -46,6 +50,7 @@ public struct CodeModeConfiguration: Sendable { ) { self.pathPolicy = pathPolicy self.networkAccessPolicy = networkAccessPolicy + self.capabilityGrant = capabilityGrant self.fileSystem = fileSystem self.artifactStore = artifactStore self.permissionBroker = permissionBroker diff --git a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift index 62cace0..428ca7d 100644 --- a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift +++ b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift @@ -42,7 +42,7 @@ public enum CodeModeAgentToolDescriptions { "items": .object([ "type": .string("string"), ]), - "description": .string("Custom provider capability keys required by the JavaScript. Built-in capability IDs may also be accepted here by hosts that expose one allowlist field."), + "description": .string("Custom provider capability keys required by the JavaScript, for example myapp.tasks.complete. Built-in capability IDs do not belong here and are ignored — list those in allowedCapabilities."), ]), "timeoutMs": .object([ "type": .string("integer"), @@ -120,11 +120,11 @@ public enum CodeModeAgentToolDescriptions { Return semantics: the runtime wraps your code in an async function. For multi-statement code, return the final graded value with an explicit top-level return statement. A script that is only a bare final top-level await expression, such as await apple.fs.read({ path: "tmp:file.txt" }), returns that awaited value. Do not use an unreturned async IIFE as the final expression. setTimeout callbacks fire synchronously in this runtime. - Allowlisting: include only the required built-in capabilities in allowedCapabilities and custom provider keys in allowedCapabilityKeys. Execution defaults to a 10000ms timeout and returns structured CodeModeToolError failures for syntax errors, missing JS helpers, runtime throws, validation failures, permission denials, timeouts, cancellation, and internal errors. + Allowlisting: include only the required built-in capabilities in allowedCapabilities and only custom provider keys in allowedCapabilityKeys. The two fields are not interchangeable: a built-in capability listed in allowedCapabilityKeys is ignored. These fields declare what your script needs; the host app applies its own ceiling on top, so a capability you list may still be withheld. Execution defaults to a 10000ms timeout and returns structured CodeModeToolError failures for syntax errors, missing JS helpers, runtime throws, validation failures, permission denials, timeouts, cancellation, and internal errors. Error repair guide: JS_API_NOT_FOUND: use the suggested JS helper names or searchJavaScriptAPI. - CAPABILITY_DENIED: add the exact capability to allowedCapabilities or allowedCapabilityKeys and retry. + CAPABILITY_DENIED: add the exact capability to allowedCapabilities (built-ins) or allowedCapabilityKeys (custom provider keys) and retry — unless the error suggestions say the host withheld it, in which case retrying will not help. PERMISSION_DENIED: the capability is allowlisted but the OS, host, or custom provider denied permission; request permission if a helper exists, otherwise tell the user or host. UI_PRESENTER_UNAVAILABLE: host configuration issue; do not retry the same call. INVALID_ARGUMENTS: use the catalog requiredArguments, optionalArguments, argumentHints, and example. diff --git a/Sources/CodeMode/Registry/CapabilityRegistry.swift b/Sources/CodeMode/Registry/CapabilityRegistry.swift index b50f76d..de09f5f 100644 --- a/Sources/CodeMode/Registry/CapabilityRegistry.swift +++ b/Sources/CodeMode/Registry/CapabilityRegistry.swift @@ -506,14 +506,13 @@ public final class CapabilityRegistry: @unchecked Sendable { return registrations[capability].map(RegisteredCodeModeFunction.init) } + /// Resolves a *custom provider* key only. Built-ins are deliberately not + /// reachable here: they are dispatched by `CapabilityID` and gated by + /// `allowedCapabilities`, so resolving them through the loosely-typed key + /// namespace would let `allowedCapabilityKeys` alias a built-in. func registeredFunction(for capabilityKey: CodeModeCapabilityKey) -> RegisteredCodeModeFunction? { lock.lock() defer { lock.unlock() } - if let builtIn = CapabilityID(rawValue: capabilityKey.rawValue), - let registration = registrations[builtIn] - { - return RegisteredCodeModeFunction(registration) - } return codeModeRegistrations[capabilityKey].map(RegisteredCodeModeFunction.init) } @@ -535,7 +534,11 @@ public final class CapabilityRegistry: @unchecked Sendable { } private func invokeBuiltIn(_ capability: CapabilityID, arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { - guard context.allowedCapabilities.contains(capability) || context.allowedCapabilityKeys.contains(capability.codeModeKey) else { + // Built-ins are gated by `allowedCapabilities` alone. `allowedCapabilityKeys` + // accepts arbitrary strings and is not validated against `CapabilityID` at + // decode time, so honouring it here would bypass any host that vets only + // the strictly-typed field. + guard context.allowedCapabilities.contains(capability) else { throw BridgeError.capabilityDenied(capability) } @@ -547,6 +550,13 @@ public final class CapabilityRegistry: @unchecked Sendable { } private func invokeCodeMode(_ capabilityKey: CodeModeCapabilityKey, arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { + // Defense in depth: `invoke` already routes anything that parses as a + // built-in to `invokeBuiltIn`, but a key that spells a built-in must never + // reach a handler through the provider path. + if let builtIn = CapabilityID(rawValue: capabilityKey.rawValue) { + throw BridgeError.capabilityDenied(builtIn) + } + guard context.allowedCapabilityKeys.contains(capabilityKey) else { throw BridgeError.capabilityKeyDenied(capabilityKey) } diff --git a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift index 4a64acf..170defb 100644 --- a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift +++ b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift @@ -2,8 +2,17 @@ import Foundation public final class BridgeInvocationContext: @unchecked Sendable { public let executionContext: ExecutionContext + /// Built-in capabilities this execution may reach: the model's declaration + /// intersected with the host's `CapabilityGrant`. public let allowedCapabilities: Set + /// Custom provider keys this execution may reach. Deliberately disjoint from + /// `allowedCapabilities` — a built-in capability is never granted by a key, or + /// `allowedCapabilityKeys` would be a way around a host that vets only the + /// strictly-typed `allowedCapabilities`. public let allowedCapabilityKeys: Set + /// Identifiers the request asked for and the host's grant withheld. Used to + /// tell the model a denial is not repairable by widening `allowedCapabilities`. + public let hostWithheldCapabilityIdentifiers: Set public let pathPolicy: any PathPolicy public let artifactStore: any ArtifactStore public let permissionBroker: any PermissionBroker @@ -19,6 +28,7 @@ public final class BridgeInvocationContext: @unchecked Sendable { executionContext: ExecutionContext, allowedCapabilities: Set, allowedCapabilityKeys: Set = [], + hostWithheldCapabilityIdentifiers: Set = [], pathPolicy: any PathPolicy, artifactStore: any ArtifactStore, permissionBroker: any PermissionBroker, @@ -29,7 +39,8 @@ public final class BridgeInvocationContext: @unchecked Sendable { ) { self.executionContext = executionContext self.allowedCapabilities = allowedCapabilities - self.allowedCapabilityKeys = allowedCapabilityKeys.union(allowedCapabilities.map(\.codeModeKey)) + self.allowedCapabilityKeys = allowedCapabilityKeys + self.hostWithheldCapabilityIdentifiers = hostWithheldCapabilityIdentifiers self.pathPolicy = pathPolicy self.artifactStore = artifactStore self.permissionBroker = permissionBroker @@ -97,6 +108,12 @@ public final class BridgeInvocationContext: @unchecked Sendable { return resolved } + /// True when the request declared this identifier but the host's + /// `CapabilityGrant` withheld it — a denial the model cannot repair. + func isWithheldByHost(_ identifier: String) -> Bool { + hostWithheldCapabilityIdentifiers.contains(identifier) + } + func recordDiagnostic(_ diagnostic: ToolDiagnostic) { transcript.record(diagnostic: diagnostic) } diff --git a/Sources/CodeMode/Runtime/BridgeRuntime.swift b/Sources/CodeMode/Runtime/BridgeRuntime.swift index 4eec3c2..5e2ad3a 100644 --- a/Sources/CodeMode/Runtime/BridgeRuntime.swift +++ b/Sources/CodeMode/Runtime/BridgeRuntime.swift @@ -182,10 +182,35 @@ final class BridgeRuntime: @unchecked Sendable { transcript: ExecutionTranscript, cancellationController: ExecutionCancellationController ) throws -> JavaScriptExecutionResult { + // `allowedCapabilities` / `allowedCapabilityKeys` are model-authored, so they + // are a declaration of intent, not a boundary. The host's grant is the + // boundary: the effective set is always requested ∩ granted. + let grant = config.capabilityGrant.resolve( + requestedCapabilities: Set(request.allowedCapabilities), + requestedCapabilityKeys: Set(request.allowedCapabilityKeys) + ) + + let withheld = grant.withheldEverything + if withheld.isEmpty == false { + transcript.record( + diagnostic: ToolDiagnostic( + severity: .warning, + code: "CAPABILITY_WITHHELD_BY_HOST", + message: "The host's capability grant withheld: \(withheld.joined(separator: ", ")). Calls needing these fail with CAPABILITY_DENIED and cannot be repaired from the script.", + category: "security", + suggestions: [ + "Do not retry with a wider allowedCapabilities; the host decides this set.", + "Complete what you can with the granted capabilities, or report the missing access to the user.", + ] + ) + ) + } + let invocationContext = BridgeInvocationContext( executionContext: request.context, - allowedCapabilities: Set(request.allowedCapabilities), - allowedCapabilityKeys: Set(request.allowedCapabilityKeys), + allowedCapabilities: grant.capabilities, + allowedCapabilityKeys: grant.capabilityKeys, + hostWithheldCapabilityIdentifiers: Set(withheld), pathPolicy: config.pathPolicy, artifactStore: config.artifactStore, permissionBroker: config.permissionBroker, @@ -256,7 +281,8 @@ final class BridgeRuntime: @unchecked Sendable { let errorPayload = self.bridgeFailurePayload( for: bridgeError, capability: capabilityID, - capabilityKey: capabilityKey + capabilityKey: capabilityKey, + invocationContext: invocationContext ) invocationContext.log(.error, message: "Capability failed \(capability): \(errorPayload.message)") invocationContext.auditLogger.log(AuditEvent(capability: capability, message: "failed: \(errorPayload.message)")) @@ -709,9 +735,15 @@ final class BridgeRuntime: @unchecked Sendable { private func bridgeFailurePayload( for error: BridgeError, capability: CapabilityID?, - capabilityKey: CodeModeCapabilityKey? + capabilityKey: CodeModeCapabilityKey?, + invocationContext: BridgeInvocationContext? = nil ) -> CodeModeToolError { - let (message, suggestions) = enrichedBridgeFailure(for: error, capability: capability, capabilityKey: capabilityKey) + let (message, suggestions) = enrichedBridgeFailure( + for: error, + capability: capability, + capabilityKey: capabilityKey, + invocationContext: invocationContext + ) return CodeModeToolError( code: error.diagnosticCode, message: message, @@ -908,20 +940,29 @@ final class BridgeRuntime: @unchecked Sendable { private func enrichedBridgeFailure( for error: BridgeError, capability: CapabilityID?, - capabilityKey: CodeModeCapabilityKey? + capabilityKey: CodeModeCapabilityKey?, + invocationContext: BridgeInvocationContext? = nil ) -> (String, [String]) { let suggestions: [String] switch error { case let .capabilityDenied(capability): - suggestions = [ - "Add \"\(capability.rawValue)\" to allowedCapabilities and retry.", - "If your host uses a unified capability-key allowlist, add \"\(capability.rawValue)\" to allowedCapabilityKeys.", - ] + bridgeSuggestions(for: capability, capabilityKey: capability.codeModeKey) + if invocationContext?.isWithheldByHost(capability.rawValue) == true { + suggestions = hostWithheldSuggestions(for: capability.rawValue) + } else { + suggestions = [ + "Add \"\(capability.rawValue)\" to allowedCapabilities and retry.", + "Built-in capabilities are granted only by allowedCapabilities; listing one in allowedCapabilityKeys has no effect.", + ] + bridgeSuggestions(for: capability, capabilityKey: capability.codeModeKey) + } case let .capabilityKeyDenied(capabilityKey): - suggestions = [ - "Add \"\(capabilityKey.rawValue)\" to allowedCapabilityKeys and retry.", - "Custom provider capabilities are not enabled by allowedCapabilities.", - ] + bridgeSuggestions(for: nil, capabilityKey: capabilityKey) + if invocationContext?.isWithheldByHost(capabilityKey.rawValue) == true { + suggestions = hostWithheldSuggestions(for: capabilityKey.rawValue) + } else { + suggestions = [ + "Add \"\(capabilityKey.rawValue)\" to allowedCapabilityKeys and retry.", + "Custom provider capabilities are not enabled by allowedCapabilities.", + ] + bridgeSuggestions(for: nil, capabilityKey: capabilityKey) + } case let .permissionDenied(permission): suggestions = permissionDeniedSuggestions(for: permission) case .customPermissionDenied: @@ -947,6 +988,14 @@ final class BridgeRuntime: @unchecked Sendable { return (error.localizedDescription, suggestions) } + private func hostWithheldSuggestions(for identifier: String) -> [String] { + [ + "The host app's capability grant does not include \"\(identifier)\".", + "This is not repaired by adding it to allowedCapabilities or allowedCapabilityKeys; the host owns this decision.", + "Continue with the capabilities you do have, or report the missing access to the user.", + ] + } + private func permissionDeniedSuggestions(for permission: PermissionKind) -> [String] { var suggestions = [ "Permission \"\(permission.rawValue)\" was denied after capability allowlisting succeeded.", diff --git a/Sources/CodeMode/Security/CapabilityGrant.swift b/Sources/CodeMode/Security/CapabilityGrant.swift new file mode 100644 index 0000000..d88e7c2 --- /dev/null +++ b/Sources/CodeMode/Security/CapabilityGrant.swift @@ -0,0 +1,76 @@ +import Foundation + +/// The host-owned ceiling on what an execution may reach. +/// +/// `JavaScriptExecutionRequest.allowedCapabilities` / `allowedCapabilityKeys` are +/// *model-authored* — they travel in the advertised tool schema, so a script +/// declares them for itself. They are a least-privilege declaration of intent and +/// a useful audit signal, but they are not a sandbox: on their own, a script that +/// asks for `keychain.read` gets it. +/// +/// `CapabilityGrant` is the half the host owns. It is configured on +/// `CodeModeConfiguration` and intersected with the request before the invocation +/// context is built, so the effective set is always +/// `requested ∩ granted`. Anything the model asks for that the grant withholds +/// fails with `CAPABILITY_DENIED` and is reported to the model as *not* +/// repairable by retrying with a wider `allowedCapabilities`. +public struct CapabilityGrant: Sendable, Equatable { + /// Built-in capabilities the host permits, or `nil` for "no host ceiling". + public var capabilities: Set? + /// Custom provider capability keys the host permits, or `nil` for "no host ceiling". + public var capabilityKeys: Set? + + public init(capabilities: Set?, capabilityKeys: Set?) { + self.capabilities = capabilities + self.capabilityKeys = capabilityKeys + } + + /// No host ceiling: whatever the request declares is what the script gets. + /// + /// This is the default for source compatibility, and it is the right choice + /// only when the host has already vetted the request itself. Hosts that pipe + /// model-authored tool JSON straight into `executeJavaScript` should set an + /// explicit grant. + public static let unrestricted = CapabilityGrant(capabilities: nil, capabilityKeys: nil) + + /// Grants nothing; every capability request is withheld. + public static let none = CapabilityGrant(capabilities: [], capabilityKeys: []) + + /// Grants exactly the listed built-in capabilities and provider keys. + public static func only( + _ capabilities: Set, + capabilityKeys: Set = [] + ) -> CapabilityGrant { + CapabilityGrant(capabilities: capabilities, capabilityKeys: capabilityKeys) + } + + /// Applies the ceiling to a model-authored request. + /// + /// Returns the effective sets alongside everything that was asked for and + /// withheld, so the caller can tell the model why a retry will not help. + public func resolve( + requestedCapabilities: Set, + requestedCapabilityKeys: Set + ) -> Resolution { + let effectiveCapabilities = capabilities.map { requestedCapabilities.intersection($0) } ?? requestedCapabilities + let effectiveKeys = capabilityKeys.map { requestedCapabilityKeys.intersection($0) } ?? requestedCapabilityKeys + + return Resolution( + capabilities: effectiveCapabilities, + capabilityKeys: effectiveKeys, + withheldCapabilities: requestedCapabilities.subtracting(effectiveCapabilities), + withheldCapabilityKeys: requestedCapabilityKeys.subtracting(effectiveKeys) + ) + } + + public struct Resolution: Sendable, Equatable { + public var capabilities: Set + public var capabilityKeys: Set + public var withheldCapabilities: Set + public var withheldCapabilityKeys: Set + + public var withheldEverything: [String] { + (withheldCapabilities.map(\.rawValue) + withheldCapabilityKeys.map(\.rawValue)).sorted() + } + } +} diff --git a/Sources/CodeModeAuthoring/README.md b/Sources/CodeModeAuthoring/README.md index 57069fa..2dbb997 100644 --- a/Sources/CodeModeAuthoring/README.md +++ b/Sources/CodeModeAuthoring/README.md @@ -69,6 +69,12 @@ let call = try await tools.executeJavaScript( ) ``` +The two allowlists are strictly disjoint: a built-in capability ID listed in +`allowedCapabilityKeys` is ignored, and a provider key listed in +`allowedCapabilities` will not decode. Both fields are model-authored, so treat +them as a declaration of intent — set `CodeModeConfiguration.capabilityGrant` for +the host-owned ceiling that actually enforces access. + Macro v1 maps one type to one JavaScript function. `Arguments` must be a nested struct, and no-arg tools use an empty `Arguments` struct. `call(arguments:)` must be `throws` or `async throws`, and it can return `Void` or a nested `Result` struct. Supported `Arguments` and `Result` property shapes are JSON primitives, `JSONValue`, arrays/dictionaries of JSON-shaped values, and optional forms. Throw `CodeModeFunctionError` for structured failures that should surface as CodeMode bridge errors. diff --git a/Tests/CodeModeTests/CodeModeProviderTests.swift b/Tests/CodeModeTests/CodeModeProviderTests.swift index 67387c9..75641a9 100644 --- a/Tests/CodeModeTests/CodeModeProviderTests.swift +++ b/Tests/CodeModeTests/CodeModeProviderTests.swift @@ -578,7 +578,11 @@ private func jsonLiteral(_ value: JSONValue) -> String { #expect(result.bool("done") == true) } -@Test func builtInCapabilitiesCanBeAllowedByCapabilityKey() async throws { +// `allowedCapabilityKeys` accepts arbitrary strings and is not validated against +// `CapabilityID` at decode time, so it must never reach a built-in bridge — a host +// that vets only the strictly-typed `allowedCapabilities` would otherwise be +// bypassed by spelling the same ID in the loose field. +@Test func builtInCapabilitiesAreNotGrantedByCapabilityKey() async throws { let (tools, sandbox) = try makeTools() defer { cleanup(sandbox) } @@ -590,9 +594,78 @@ private func jsonLiteral(_ value: JSONValue) -> String { ) ) let observed = await observe(call) + let error = try #require(observed.error) + #expect(error.code == "CAPABILITY_DENIED") + #expect(error.capabilityKey == CapabilityID.keychainRead.codeModeKey) +} + +@Test func builtInCapabilitiesRemainGrantedByCapabilityID() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await apple.keychain.get("missing-provider-key-test");"#, + allowedCapabilities: [.keychainRead] + ) + ) + let observed = await observe(call) #expect(observed.result?.output == .null) } +@Test func hostCapabilityGrantOverridesModelAuthoredAllowlist() async throws { + let (tools, sandbox) = try makeTools( + capabilityGrant: .only([.fsRead], capabilityKeys: []), + codeModeProviders: [ManualProvider()] + ) + defer { cleanup(sandbox) } + + // The script asks for keychain.read and a provider key; the host grants neither. + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await apple.keychain.get("anything");"#, + allowedCapabilities: [.keychainRead], + allowedCapabilityKeys: ["myapp.api.doTheThing"] + ) + ) + let observed = await observe(call) + let error = try #require(observed.error) + #expect(error.code == "CAPABILITY_DENIED") + #expect(error.suggestions.contains { $0.contains("host app's capability grant") }) + #expect( + error.diagnostics.contains { $0.code == "CAPABILITY_WITHHELD_BY_HOST" } + ) + + let providerCall = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.api.doTheThing({ id: "123" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.api.doTheThing"] + ) + ) + let providerObserved = await observe(providerCall) + #expect(providerObserved.error?.code == "CAPABILITY_DENIED") +} + +@Test func hostCapabilityGrantPermitsWhatItIncludes() async throws { + let (tools, sandbox) = try makeTools( + capabilityGrant: .only([], capabilityKeys: ["myapp.api.doTheThing"]), + codeModeProviders: [ManualProvider()] + ) + defer { cleanup(sandbox) } + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.api.doTheThing({ id: "123" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.api.doTheThing"] + ) + ) + let observed = await observe(call) + let result = try #require(observed.result?.output?.objectValue) + #expect(result.string("id") == "123") +} + @Test func customProviderFunctionErrorsRemainStructured() async throws { let (tools, sandbox) = try makeTools(codeModeProviders: [ManualProvider()]) defer { cleanup(sandbox) } diff --git a/Tests/CodeModeTests/TestSupport.swift b/Tests/CodeModeTests/TestSupport.swift index 3e2a23a..8c31603 100644 --- a/Tests/CodeModeTests/TestSupport.swift +++ b/Tests/CodeModeTests/TestSupport.swift @@ -35,6 +35,7 @@ func cleanup(_ sandbox: TestSandbox) { func makeTools( permissionBroker: any PermissionBroker = NoopPermissionBroker(), + capabilityGrant: CapabilityGrant = .unrestricted, fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), systemUIPresenter: any SystemUIPresenter = UnavailableSystemUIPresenter(), eventInbox: any CodeModeEventInbox = UnavailableCodeModeEventInbox(), @@ -59,6 +60,7 @@ func makeTools( let configuration = CodeModeConfiguration( pathPolicy: pathPolicy, + capabilityGrant: capabilityGrant, fileSystem: fileSystem, artifactStore: InMemoryArtifactStore(), permissionBroker: permissionBroker, From 6f0e702b618262b5b60eac794fb8a8ea9c3973e9 Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 19:45:18 -0700 Subject: [PATCH 02/18] Make adversarial number handling total instead of trapping `Int.init(_: Double)` traps on non-finite and out-of-range input, and every number reaching these paths is attacker-controlled: `JSONValue.intValue` backs 48 `.int(...)` call sites, and `JavaScriptExecutionRequest.decodeTimeoutMs` runs on model-authored tool JSON before any script executes. `{"timeoutMs": 1e300}` or `fetch(url, { timeoutMs: 1e300 })` killed the host process in-process. - `JSONValue.intValue` goes through a total `exactInt(from:)` that rejects NaN, infinity, and out-of-range magnitudes; in-range values truncate toward zero as before. - `decodeTimeoutMs` rejects doubles and numeric strings it cannot represent. - `timeoutMs` is clamped to the 1...60000 bounds the tool schema already advertises, in both the memberwise init and the decoder, so an over-large budget yields the ceiling rather than an unbounded run. - A declared-`.number` argument must now be finite: the type check rejects non-finite JSON numbers, and `coerceScalarString` no longer accepts `"inf"`/`"nan"`, which `Double.init(String)` parses happily. Review finding P0 #3. --- Sources/CodeMode/API/BridgeModels.swift | 38 ++++++++-- .../Registry/CapabilityRegistry.swift | 10 ++- Sources/CodeMode/Support/JSONValue.swift | 23 +++++- .../DefensiveDecodingTests.swift | 76 +++++++++++++++++++ 4 files changed, 139 insertions(+), 8 deletions(-) diff --git a/Sources/CodeMode/API/BridgeModels.swift b/Sources/CodeMode/API/BridgeModels.swift index 3906e2f..03d8002 100644 --- a/Sources/CodeMode/API/BridgeModels.swift +++ b/Sources/CodeMode/API/BridgeModels.swift @@ -137,6 +137,13 @@ public struct JavaScriptAPISearchResponse: Sendable, Codable, Equatable { } public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { + /// Bounds advertised in `executeJavaScriptParameterSchema`. Values outside + /// them are clamped rather than rejected: a model that asks for a 10-minute + /// budget should get the ceiling and a running script, not a hard failure. + public static let minimumTimeoutMs = 1 + public static let maximumTimeoutMs = 60_000 + public static let defaultTimeoutMs = 10_000 + public var code: String public var allowedCapabilities: [CapabilityID] public var allowedCapabilityKeys: [CodeModeCapabilityKey] @@ -147,16 +154,20 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { code: String, allowedCapabilities: [CapabilityID], allowedCapabilityKeys: [CodeModeCapabilityKey] = [], - timeoutMs: Int = 10_000, + timeoutMs: Int = JavaScriptExecutionRequest.defaultTimeoutMs, context: ExecutionContext = .init() ) { self.code = code self.allowedCapabilities = allowedCapabilities self.allowedCapabilityKeys = allowedCapabilityKeys - self.timeoutMs = timeoutMs + self.timeoutMs = Self.clampTimeoutMs(timeoutMs) self.context = context } + static func clampTimeoutMs(_ value: Int) -> Int { + min(max(value, minimumTimeoutMs), maximumTimeoutMs) + } + private enum CodingKeys: String, CodingKey { case code case allowedCapabilities @@ -199,10 +210,13 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { self.allowedCapabilities = capabilities self.allowedCapabilityKeys = try container.decodeIfPresent([CodeModeCapabilityKey].self, forKey: .allowedCapabilityKeys) ?? [] - self.timeoutMs = try Self.decodeTimeoutMs(from: container) ?? 10_000 + self.timeoutMs = Self.clampTimeoutMs(try Self.decodeTimeoutMs(from: container) ?? Self.defaultTimeoutMs) self.context = try container.decodeIfPresent(ExecutionContext.self, forKey: .context) ?? .init() } + // Every conversion here is total. `Int.init(_: Double)` traps on non-finite + // and out-of-range input, and this runs on model-authored JSON before any + // script does — `{"timeoutMs": 1e300}` would kill the host process in-process. private static func decodeTimeoutMs(from container: KeyedDecodingContainer) throws -> Int? { guard container.contains(.timeoutMs), try !container.decodeNil(forKey: .timeoutMs) else { return nil @@ -211,11 +225,11 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { return value } if let value = try? container.decode(Double.self, forKey: .timeoutMs) { - return Int(value) + return try requireRepresentable(value, in: container) } if let string = try? container.decode(String.self, forKey: .timeoutMs), let value = Double(string.trimmingCharacters(in: .whitespaces)) { - return Int(value) + return try requireRepresentable(value, in: container) } throw DecodingError.dataCorruptedError( forKey: .timeoutMs, @@ -223,6 +237,20 @@ public struct JavaScriptExecutionRequest: Sendable, Codable, Equatable { debugDescription: "timeoutMs must be an integer or a numeric string." ) } + + private static func requireRepresentable( + _ value: Double, + in container: KeyedDecodingContainer + ) throws -> Int { + guard let converted = JSONValue.exactInt(from: value) else { + throw DecodingError.dataCorruptedError( + forKey: .timeoutMs, + in: container, + debugDescription: "timeoutMs must be a finite number representable as an integer; received \(value)." + ) + } + return converted + } } public enum JavaScriptExecutionEvent: Sendable, Equatable { diff --git a/Sources/CodeMode/Registry/CapabilityRegistry.swift b/Sources/CodeMode/Registry/CapabilityRegistry.swift index de09f5f..a6d16b5 100644 --- a/Sources/CodeMode/Registry/CapabilityRegistry.swift +++ b/Sources/CodeMode/Registry/CapabilityRegistry.swift @@ -16,7 +16,10 @@ public enum CapabilityArgumentType: String, Sendable, Codable, Equatable { if case .string = value { return true } return false case .number: - if case .number = value { return true } + // Non-finite is not a usable number for any bridge: it cannot be + // converted to an Int, cannot round-trip through JSON, and reaches + // here from `"inf"`/`"nan"` coercion or a JS `Infinity`. + if case let .number(number) = value { return number.isFinite } return false case .bool: if case .bool = value { return true } @@ -604,7 +607,10 @@ public final class CapabilityRegistry: @unchecked Sendable { let trimmed = raw.trimmingCharacters(in: .whitespaces) switch type { case .number: - guard let number = Double(trimmed) else { return nil } + // `Double.init(String)` accepts "inf", "-infinity", and "nan", so a + // declared-number argument sent as a string is a way to smuggle a + // non-finite value past the type check. + guard let number = Double(trimmed), number.isFinite else { return nil } return .number(number) case .bool: switch trimmed.lowercased() { diff --git a/Sources/CodeMode/Support/JSONValue.swift b/Sources/CodeMode/Support/JSONValue.swift index d96eec3..0442e13 100644 --- a/Sources/CodeMode/Support/JSONValue.swift +++ b/Sources/CodeMode/Support/JSONValue.swift @@ -131,13 +131,34 @@ public extension JSONValue { return nil } + /// Integer view of a JSON number, or `nil` when the value cannot be + /// represented. + /// + /// `Int.init(_: Double)` *traps* on NaN, infinity, and anything outside + /// `Int`'s range, and every value here came from adversarial JSON — a model + /// tool call or a script's `fetch(url, { timeoutMs: 1e300 })`. The conversion + /// has to be total or a single number kills the host process. var intValue: Int? { if case let .number(value) = self { - return Int(value) + return JSONValue.exactInt(from: value) } return nil } + /// Total `Double` → `Int` conversion. Rejects non-finite values and anything + /// the conversion cannot represent; truncates toward zero otherwise, matching + /// the previous behavior for in-range input. + static func exactInt(from value: Double) -> Int? { + guard value.isFinite else { return nil } + let truncated = value.rounded(.towardZero) + guard truncated >= -9_223_372_036_854_775_808.0, + truncated < 9_223_372_036_854_775_808.0 + else { + return nil + } + return Int(truncated) + } + var doubleValue: Double? { if case let .number(value) = self { return value diff --git a/Tests/CodeModeTests/DefensiveDecodingTests.swift b/Tests/CodeModeTests/DefensiveDecodingTests.swift index 8a4485c..e9ede14 100644 --- a/Tests/CodeModeTests/DefensiveDecodingTests.swift +++ b/Tests/CodeModeTests/DefensiveDecodingTests.swift @@ -56,6 +56,50 @@ private func decodeRequest(_ json: String) throws -> JavaScriptExecutionRequest } } +// MARK: - Adversarial numbers must not trap + +@Test func executionRequestRejectsNonRepresentableTimeout() throws { + // `Int.init(_: Double)` traps on these. This decode path runs on + // model-authored JSON before any script does, so a trap here is an + // in-process kill of the host app. + for literal in ["1e300", "-1e300", "1e400", "-1e400"] { + #expect(throws: (any Error).self) { + _ = try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":\#(literal)}"#) + } + #expect(throws: (any Error).self) { + _ = try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":"\#(literal)"}"#) + } + } + + for literal in ["inf", "-inf", "infinity", "nan", "NaN"] { + #expect(throws: (any Error).self) { + _ = try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":"\#(literal)"}"#) + } + } +} + +@Test func executionRequestClampsTimeoutToAdvertisedBounds() throws { + // The schema advertises 1...60000; out-of-range values are clamped rather + // than failing the call. + #expect(try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":600000}"#).timeoutMs == 60_000) + #expect(try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":0}"#).timeoutMs == 1) + #expect(try decodeRequest(#"{"code":"return 1;","allowedCapabilities":[],"timeoutMs":-5}"#).timeoutMs == 1) + #expect(JavaScriptExecutionRequest(code: "", allowedCapabilities: [], timeoutMs: 10_000_000).timeoutMs == 60_000) +} + +@Test func jsonValueIntConversionIsTotal() { + #expect(JSONValue.number(.infinity).intValue == nil) + #expect(JSONValue.number(-.infinity).intValue == nil) + #expect(JSONValue.number(.nan).intValue == nil) + #expect(JSONValue.number(1e300).intValue == nil) + #expect(JSONValue.number(-1e300).intValue == nil) + #expect(JSONValue.number(9.3e18).intValue == nil) + #expect(JSONValue.number(42).intValue == 42) + #expect(JSONValue.number(-42.9).intValue == -42) + #expect(JSONValue.number(Double(Int.max)).intValue == nil) + #expect(JSONValue.number(Double(Int.min)).intValue == Int.min) +} + @Test func executionRequestRoundTripsThroughCodable() throws { let original = JavaScriptExecutionRequest( code: "return 1;", @@ -118,6 +162,38 @@ private func makeEchoRegistry(argumentTypes: [String: CapabilityArgumentType]) - } } +@Test func registryRejectsNonFiniteDeclaredNumbers() throws { + let registry = makeEchoRegistry(argumentTypes: ["timeoutMs": .number]) + let (context, sandbox) = try makeInvocationContext(allowedCapabilities: [.fsRead]) + defer { cleanup(sandbox) } + + // Straight through as a JSON number... + #expect(throws: (any Error).self) { + _ = try registry.invoke( + CapabilityID.fsRead.rawValue, + arguments: ["timeoutMs": .number(.infinity)], + context: context + ) + } + // ...and smuggled through string coercion, which `Double.init(String)` accepts. + for spelling in ["inf", "-infinity", "nan"] { + #expect(throws: (any Error).self) { + _ = try registry.invoke( + CapabilityID.fsRead.rawValue, + arguments: ["timeoutMs": .string(spelling)], + context: context + ) + } + } + + let ok = try registry.invoke( + CapabilityID.fsRead.rawValue, + arguments: ["timeoutMs": .string("2500")], + context: context + ) + #expect(ok.objectValue?["timeoutMs"] == .number(2_500)) +} + @Test func registryDoesNotCoerceNonBooleanStringsOrBadNumbers() throws { let registry = makeEchoRegistry(argumentTypes: ["count": .number, "flag": .bool]) let (context, sandbox) = try makeInvocationContext(allowedCapabilities: [.fsRead]) From a6f866d334388be9e147268908d231f483dfa816 Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 19:50:45 -0700 Subject: [PATCH 03/18] Stop fs.move/fs.copy from recursively deleting their destination MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both operations unconditionally recursive-deleted any existing destination, with no overwrite flag and no directory check, and containment alone admits a root: `DefaultPathPolicy.resolve(path: "documents:")` yields the documents root and `isAllowed` accepts it via `path == allowed`. So `apple.fs.move({ from: 'tmp:junk.txt', to: 'documents:' })` removed the user's entire Documents directory. `fs.delete` already gated directories correctly, so the three operations disagreed. Destinations now go through one guard shared by move and copy: a sandbox root is refused outright, an existing destination requires `overwrite: true`, and an existing *directory* additionally requires `recursive: true` — the same shape `fs.delete` enforces. `fs.delete` picks up the root check too, since `{path: 'documents:', recursive: true}` had the same reach. Also bounds and clarifies file IO: - `CodeModeConfiguration.fileSystemLimits` caps `fs.read`/`fs.write` (16 MB default). A 2 GB file in `documents:` was previously read whole and base64-serialized. The read cap is checked against stat before the read, so it does not spend the memory it exists to protect; the write cap is checked before the parent directory is created, so a refused write leaves nothing behind. - A `utf8` read of undecodable bytes now fails with a repair suggestion instead of silently returning `""`, which told the script the file was empty. - `encoding` is a `CodeModeStringEnum`, so the catalog advertises `allowedStringValues` instead of leaving the constraint in prose. `PathPolicy` gains `allowedRoots` (defaulted empty, so custom policies still compile) to make "is this a root" answerable. Golden metadata baseline regenerated; the diff is the four fs entries above. Review finding P0 #4, plus the `fs.read` encoding-constraint item from P2. --- Sources/CodeMode/API/BridgeModels.swift | 4 + .../Bridges/DefaultCapabilityLoader.swift | 5 +- .../CodeMode/Bridges/FileSystemBridge.swift | 122 ++++++++++-- .../Bridges/FileSystemCodeModeTools.swift | 35 +++- .../CodeMode/Host/CodeModeAgentTools.swift | 1 + Sources/CodeMode/Security/PathPolicy.swift | 24 +++ .../CodeModeTests/FileSystemBridgeTests.swift | 185 ++++++++++++++++++ .../capability-metadata-golden.json | 36 +++- 8 files changed, 378 insertions(+), 34 deletions(-) diff --git a/Sources/CodeMode/API/BridgeModels.swift b/Sources/CodeMode/API/BridgeModels.swift index 03d8002..b5c6caa 100644 --- a/Sources/CodeMode/API/BridgeModels.swift +++ b/Sources/CodeMode/API/BridgeModels.swift @@ -7,6 +7,8 @@ public struct CodeModeConfiguration: Sendable { /// `allowedCapabilityKeys`. Defaults to `.unrestricted`; see `CapabilityGrant`. public var capabilityGrant: CapabilityGrant public var fileSystem: any CodeModeFileSystem + /// Byte ceilings applied by the filesystem bridge to `fs.read` / `fs.write`. + public var fileSystemLimits: FileSystemLimits public var artifactStore: any ArtifactStore public var permissionBroker: any PermissionBroker public var auditLogger: any AuditLogger @@ -30,6 +32,7 @@ public struct CodeModeConfiguration: Sendable { networkAccessPolicy: NetworkAccessPolicy = .standard, capabilityGrant: CapabilityGrant = .unrestricted, fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), + fileSystemLimits: FileSystemLimits = .standard, artifactStore: any ArtifactStore = InMemoryArtifactStore(), permissionBroker: any PermissionBroker = SystemPermissionBroker(), auditLogger: any AuditLogger = SyncAuditLogger(), @@ -52,6 +55,7 @@ public struct CodeModeConfiguration: Sendable { self.networkAccessPolicy = networkAccessPolicy self.capabilityGrant = capabilityGrant self.fileSystem = fileSystem + self.fileSystemLimits = fileSystemLimits self.artifactStore = artifactStore self.permissionBroker = permissionBroker self.auditLogger = auditLogger diff --git a/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift b/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift index 96eb81e..187308d 100644 --- a/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift +++ b/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift @@ -3,6 +3,7 @@ import Foundation public enum DefaultCapabilityLoader { public static func loadAllRegistrations( fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), + fileSystemLimits: FileSystemLimits = .standard, networkAccessPolicy: NetworkAccessPolicy = .standard, eventInbox: any CodeModeEventInbox = UnavailableCodeModeEventInbox(), cloudKitClient: any CloudKitClient = UnavailableCloudKitClient(), @@ -18,6 +19,7 @@ public enum DefaultCapabilityLoader { ) -> [CapabilityRegistration] { DefaultCapabilityRegistrationBuilder( fileSystem: fileSystem, + fileSystemLimits: fileSystemLimits, networkAccessPolicy: networkAccessPolicy, eventInbox: eventInbox, cloudKitClient: cloudKitClient, @@ -64,6 +66,7 @@ struct DefaultCapabilityRegistrationBuilder { init( fileSystem: any CodeModeFileSystem, + fileSystemLimits: FileSystemLimits, networkAccessPolicy: NetworkAccessPolicy, eventInbox: any CodeModeEventInbox, cloudKitClient: any CloudKitClient, @@ -77,7 +80,7 @@ struct DefaultCapabilityRegistrationBuilder { passKitClient: any PassKitClient, storeKitClient: any StoreKitClient ) { - self.fs = FileSystemBridge(fileSystem: fileSystem) + self.fs = FileSystemBridge(fileSystem: fileSystem, limits: fileSystemLimits) self.network = NetworkBridge(policy: networkAccessPolicy) self.keychain = KeychainBridge() self.location = LocationBridge() diff --git a/Sources/CodeMode/Bridges/FileSystemBridge.swift b/Sources/CodeMode/Bridges/FileSystemBridge.swift index a4b8be9..4ba8b25 100644 --- a/Sources/CodeMode/Bridges/FileSystemBridge.swift +++ b/Sources/CodeMode/Bridges/FileSystemBridge.swift @@ -1,14 +1,40 @@ import Foundation +/// Byte ceilings for the filesystem bridge. +/// +/// A sandbox root can hold files far larger than a script's result budget — a +/// 2 GB video in `documents:` read whole and base64-serialized is an immediate +/// jetsam on iOS — so reads and writes are bounded and fail with a diagnostic +/// instead of exhausting memory. +public struct FileSystemLimits: Sendable, Equatable { + public var maxReadBytes: Int + public var maxWriteBytes: Int + + public init(maxReadBytes: Int = 16 * 1_024 * 1_024, maxWriteBytes: Int = 16 * 1_024 * 1_024) { + self.maxReadBytes = maxReadBytes + self.maxWriteBytes = maxWriteBytes + } + + public static let standard = FileSystemLimits() + + /// No ceiling. Only appropriate when the host has its own quota. + public static let unlimited = FileSystemLimits(maxReadBytes: .max, maxWriteBytes: .max) +} + public final class FileSystemBridge: @unchecked Sendable { private let fileSystem: any CodeModeFileSystem + private let limits: FileSystemLimits - public init(fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem()) { + public init( + fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), + limits: FileSystemLimits = .standard + ) { self.fileSystem = fileSystem + self.limits = limits } - public convenience init(fileManager: FileManager) { - self.init(fileSystem: LocalCodeModeFileSystem(fileManager: fileManager)) + public convenience init(fileManager: FileManager, limits: FileSystemLimits = .standard) { + self.init(fileSystem: LocalCodeModeFileSystem(fileManager: fileManager), limits: limits) } public func list(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { @@ -36,11 +62,31 @@ public final class FileSystemBridge: @unchecked Sendable { let encoding = arguments.string("encoding") ?? "utf8" let url = try context.pathPolicy.resolve(path: path) + + // Check the size before reading: reading first and then rejecting has + // already spent the memory the cap exists to protect. + if let size = try? fileSystem.attributesOfItem(at: url).size, size > limits.maxReadBytes { + throw BridgeError.invalidArguments( + "fs.read refused \(url.lastPathComponent): \(size) bytes exceeds the \(limits.maxReadBytes)-byte read limit. Read a smaller file, or have the host raise CodeModeConfiguration.fileSystemLimits." + ) + } + let data = try fileSystem.readData(at: url) + guard data.count <= limits.maxReadBytes else { + throw BridgeError.invalidArguments( + "fs.read refused \(url.lastPathComponent): \(data.count) bytes exceeds the \(limits.maxReadBytes)-byte read limit." + ) + } switch encoding.lowercased() { case "utf8", "utf-8": - let text = String(data: data, encoding: .utf8) ?? "" + // Silently substituting "" for undecodable bytes tells the script the + // file is empty, which is worse than a failure it can act on. + guard let text = String(data: data, encoding: .utf8) else { + throw BridgeError.invalidArguments( + "fs.read could not decode \(url.lastPathComponent) as UTF-8. Retry with encoding: 'base64' for binary data." + ) + } return .object([ "path": .string(url.path), "text": .string(text), @@ -63,9 +109,6 @@ public final class FileSystemBridge: @unchecked Sendable { let encoding = arguments.string("encoding") ?? "utf8" let url = try context.pathPolicy.resolve(path: path) - let parent = url.deletingLastPathComponent() - try fileSystem.createDirectory(at: parent, recursive: true) - let data: Data switch encoding.lowercased() { case "utf8", "utf-8": @@ -80,6 +123,15 @@ public final class FileSystemBridge: @unchecked Sendable { throw BridgeError.invalidArguments("Unsupported encoding: \(encoding)") } + guard data.count <= limits.maxWriteBytes else { + throw BridgeError.invalidArguments( + "fs.write refused \(url.lastPathComponent): \(data.count) bytes exceeds the \(limits.maxWriteBytes)-byte write limit." + ) + } + + // Only after the payload is known good, so a rejected write leaves no + // directories behind. + try fileSystem.createDirectory(at: url.deletingLastPathComponent(), recursive: true) try fileSystem.writeData(data, to: url) return .object([ "path": .string(url.path), @@ -94,10 +146,7 @@ public final class FileSystemBridge: @unchecked Sendable { let fromURL = try context.pathPolicy.resolve(path: from) let toURL = try context.pathPolicy.resolve(path: to) - - if fileSystem.itemExists(at: toURL) { - try fileSystem.removeItem(at: toURL) - } + try prepareDestination(toURL, operation: "fs.move", arguments: arguments, context: context) try fileSystem.moveItem(at: fromURL, to: toURL) return .object([ @@ -113,10 +162,7 @@ public final class FileSystemBridge: @unchecked Sendable { let fromURL = try context.pathPolicy.resolve(path: from) let toURL = try context.pathPolicy.resolve(path: to) - - if fileSystem.itemExists(at: toURL) { - try fileSystem.removeItem(at: toURL) - } + try prepareDestination(toURL, operation: "fs.copy", arguments: arguments, context: context) try fileSystem.copyItem(at: fromURL, to: toURL) return .object([ @@ -125,6 +171,46 @@ public final class FileSystemBridge: @unchecked Sendable { ]) } + /// Clears the way for a move/copy, refusing anything destructive the caller + /// did not explicitly ask for. + /// + /// The previous behavior was an unconditional recursive `removeItem` on any + /// existing destination, and containment alone admits a root: `documents:` + /// resolves to the documents root, so `fs.move({ from: 'tmp:junk.txt', + /// to: 'documents:' })` deleted the user's whole Documents directory. The + /// gates here match the ones `fs.delete` already enforces. + private func prepareDestination( + _ toURL: URL, + operation: String, + arguments: [String: JSONValue], + context: BridgeInvocationContext + ) throws { + guard context.pathPolicy.isAllowedRoot(toURL) == false else { + throw BridgeError.pathViolation( + "\(operation) refuses a sandbox root as its destination. Name a path inside the root, for example 'documents:archive/file.txt'." + ) + } + + guard fileSystem.itemExists(at: toURL) else { + return + } + + guard arguments.bool("overwrite") == true else { + throw BridgeError.invalidArguments( + "\(operation) destination already exists. Pass overwrite: true to replace it, or choose a different 'to' path." + ) + } + + let isDirectory = (try? fileSystem.attributesOfItem(at: toURL))?.isDirectory ?? false + if isDirectory, arguments.bool("recursive") != true { + throw BridgeError.invalidArguments( + "\(operation) destination is a directory. Pass recursive: true along with overwrite: true to replace it and everything under it." + ) + } + + try fileSystem.removeItem(at: toURL) + } + public func delete(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { guard let path = arguments.string("path") else { throw BridgeError.invalidArguments("fs.delete requires 'path'") @@ -137,6 +223,12 @@ public final class FileSystemBridge: @unchecked Sendable { return .object(["deleted": .bool(false), "path": .string(url.path)]) } + guard context.pathPolicy.isAllowedRoot(url) == false else { + throw BridgeError.pathViolation( + "fs.delete refuses a sandbox root. Delete entries inside it individually." + ) + } + let attrs = try fileSystem.attributesOfItem(at: url) if attrs.isDirectory, recursive == false { throw BridgeError.invalidArguments("fs.delete requires recursive=true for directories") diff --git a/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift b/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift index 650190b..ca296cb 100644 --- a/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift +++ b/Sources/CodeMode/Bridges/FileSystemCodeModeTools.swift @@ -1,7 +1,16 @@ import Foundation -// Filesystem capabilities. Flat arguments, no constrained values, so no -// CodeModeStringEnum is needed here. +// Filesystem capabilities. + +/// The encodings `fs.read`/`fs.write` accept. Declared as an enum so the catalog +/// advertises `allowedStringValues` instead of leaving the constraint in prose, +/// where the model cannot discover it. +enum FileEncoding: String, CodeModeStringEnum { + case utf8 + case base64 + + static let codeModeAliases: [String: FileEncoding] = ["utf-8": .utf8] +} @BuiltInCodeMode(.fsList, path: "apple.fs.list", aliases: ["fs.promises.readdir"]) struct FSListTool: BuiltInCodeModeTool { @@ -35,8 +44,8 @@ struct FSReadTool: BuiltInCodeModeTool { struct Arguments: Sendable { @ToolParam("File path using allowed root prefix.") var path: String - @ToolParam("utf8 (default) or base64.") - var encoding: String? + @ToolParam("utf8 (default) or base64. Binary files must use base64; a utf8 read of undecodable bytes fails rather than returning empty text.") + var encoding: FileEncoding? var raw: [String: JSONValue] } @@ -61,7 +70,7 @@ struct FSWriteTool: BuiltInCodeModeTool { @ToolParam("UTF-8 text or base64 string depending on encoding.") var data: String? @ToolParam("utf8 (default) or base64.") - var encoding: String? + var encoding: FileEncoding? var raw: [String: JSONValue] } @@ -75,7 +84,7 @@ struct FSWriteTool: BuiltInCodeModeTool { @BuiltInCodeMode(.fsMove, path: "apple.fs.move", aliases: ["fs.promises.rename"]) struct FSMoveTool: BuiltInCodeModeTool { static let codeModeTitle = "Move file" - static let codeModeSummary = "Move file/directory within allowed sandbox roots." + static let codeModeSummary = "Move file/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination." static let codeModeTags = ["filesystem", "io", "fs"] static let codeModeExample = "await apple.fs.move({ from: 'tmp:a.txt', to: 'tmp:b.txt' })" static let codeModeResultSummary = "Object with from/to resolved paths." @@ -83,8 +92,12 @@ struct FSMoveTool: BuiltInCodeModeTool { struct Arguments: Sendable { @ToolParam("Source sandbox path.") var from: String - @ToolParam("Destination sandbox path.") + @ToolParam("Destination sandbox path. Must name a path inside a root, not the root itself.") var to: String + @ToolParam("Required as true to replace an existing destination; default false.") + var overwrite: Bool? + @ToolParam("Required as true alongside overwrite when the existing destination is a directory.") + var recursive: Bool? var raw: [String: JSONValue] } @@ -98,7 +111,7 @@ struct FSMoveTool: BuiltInCodeModeTool { @BuiltInCodeMode(.fsCopy, path: "apple.fs.copy", aliases: ["fs.promises.copyFile"]) struct FSCopyTool: BuiltInCodeModeTool { static let codeModeTitle = "Copy file" - static let codeModeSummary = "Copy file/directory within allowed sandbox roots." + static let codeModeSummary = "Copy file/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination." static let codeModeTags = ["filesystem", "io", "fs"] static let codeModeExample = "await apple.fs.copy({ from: 'tmp:a.txt', to: 'tmp:b.txt' })" static let codeModeResultSummary = "Object with from/to resolved paths." @@ -106,8 +119,12 @@ struct FSCopyTool: BuiltInCodeModeTool { struct Arguments: Sendable { @ToolParam("Source sandbox path.") var from: String - @ToolParam("Destination sandbox path.") + @ToolParam("Destination sandbox path. Must name a path inside a root, not the root itself.") var to: String + @ToolParam("Required as true to replace an existing destination; default false.") + var overwrite: Bool? + @ToolParam("Required as true alongside overwrite when the existing destination is a directory.") + var recursive: Bool? var raw: [String: JSONValue] } diff --git a/Sources/CodeMode/Host/CodeModeAgentTools.swift b/Sources/CodeMode/Host/CodeModeAgentTools.swift index 1959dfb..a269604 100644 --- a/Sources/CodeMode/Host/CodeModeAgentTools.swift +++ b/Sources/CodeMode/Host/CodeModeAgentTools.swift @@ -8,6 +8,7 @@ public final class CodeModeAgentTools: @unchecked Sendable { public init(config: CodeModeConfiguration = .init()) { let allDefaultRegistrations = DefaultCapabilityLoader.loadAllRegistrations( fileSystem: config.fileSystem, + fileSystemLimits: config.fileSystemLimits, networkAccessPolicy: config.networkAccessPolicy, eventInbox: config.eventInbox, cloudKitClient: config.cloudKitClient, diff --git a/Sources/CodeMode/Security/PathPolicy.swift b/Sources/CodeMode/Security/PathPolicy.swift index 6ba31f8..08af30b 100644 --- a/Sources/CodeMode/Security/PathPolicy.swift +++ b/Sources/CodeMode/Security/PathPolicy.swift @@ -23,6 +23,26 @@ public struct PathPolicyConfig: Sendable { public protocol PathPolicy: Sendable { func resolve(path: String) throws -> URL + + /// The roots this policy admits. + /// + /// `resolve` accepts a root itself — `documents:` resolves to the documents + /// root — so destructive operations need to be able to recognize one and + /// refuse. Custom policies that do not implement this get an empty list and + /// simply lose that specific check. + var allowedRoots: [URL] { get } +} + +public extension PathPolicy { + var allowedRoots: [URL] { [] } + + /// True when `url` *is* one of the allowed roots rather than something inside one. + func isAllowedRoot(_ url: URL) -> Bool { + let candidate = url.standardizedFileURL.resolvingSymlinksInPath().standardizedFileURL.path + return allowedRoots.contains { root in + root.standardizedFileURL.resolvingSymlinksInPath().standardizedFileURL.path == candidate + } + } } public struct DefaultPathPolicy: PathPolicy { @@ -32,6 +52,10 @@ public struct DefaultPathPolicy: PathPolicy { self.config = config } + public var allowedRoots: [URL] { + [config.tmpRoot, config.cachesRoot, config.documentsRoot, config.appGroupRoot].compactMap { $0 } + } + public func resolve(path: String) throws -> URL { let cleaned = path.trimmingCharacters(in: .whitespacesAndNewlines) guard cleaned.isEmpty == false else { diff --git a/Tests/CodeModeTests/FileSystemBridgeTests.swift b/Tests/CodeModeTests/FileSystemBridgeTests.swift index 2cae75c..f7482ac 100644 --- a/Tests/CodeModeTests/FileSystemBridgeTests.swift +++ b/Tests/CodeModeTests/FileSystemBridgeTests.swift @@ -239,3 +239,188 @@ private final class RecordingCodeModeFileSystem: CodeModeFileSystem, @unchecked let payload = try requireJSONObject(from: try #require(observed.result)) #expect(payload["text"] as? String == "fs-from-execute") } + +// MARK: - Destructive-destination guards + +@Test func fileSystemMoveRefusesASandboxRootAsDestination() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:junk.txt"), "data": .string("x")], context: context) + _ = try fs.write(arguments: ["path": .string("documents:keepme.txt"), "data": .string("precious")], context: context) + + // `documents:` resolves to the documents root itself, and containment admits + // it. The old unconditional removeItem here deleted the user's Documents. + for destination in ["documents:", "documents:/", "tmp:", "caches:"] { + do { + _ = try fs.move(arguments: [ + "from": .string("tmp:junk.txt"), + "to": .string(destination), + "overwrite": .bool(true), + "recursive": .bool(true), + ], context: context) + Issue.record("Expected fs.move to refuse the root destination \(destination)") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } + } + + let survived = try fs.read(arguments: ["path": .string("documents:keepme.txt")], context: context) + #expect(try requireObject(survived).string("text") == "precious") +} + +@Test func fileSystemCopyRefusesASandboxRootAsDestination() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:junk.txt"), "data": .string("x")], context: context) + + #expect(throws: (any Error).self) { + _ = try fs.copy(arguments: [ + "from": .string("tmp:junk.txt"), + "to": .string("documents:"), + "overwrite": .bool(true), + ], context: context) + } +} + +@Test func fileSystemDeleteRefusesASandboxRoot() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + do { + _ = try fs.delete(arguments: ["path": .string("documents:"), "recursive": .bool(true)], context: context) + Issue.record("Expected fs.delete to refuse a sandbox root") + } catch { + #expect(requireBridgeErrorCode(error) == "PATH_POLICY_VIOLATION") + } + #expect(FileManager.default.fileExists(atPath: sandbox.documents.path)) +} + +@Test func fileSystemMoveAndCopyRequireExplicitOverwrite() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:a.txt"), "data": .string("source")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:b.txt"), "data": .string("destination")], context: context) + + do { + _ = try fs.copy(arguments: ["from": .string("tmp:a.txt"), "to": .string("tmp:b.txt")], context: context) + Issue.record("Expected fs.copy to refuse an existing destination without overwrite") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + + // The destination is untouched by the refused call. + let intact = try fs.read(arguments: ["path": .string("tmp:b.txt")], context: context) + #expect(try requireObject(intact).string("text") == "destination") + + _ = try fs.copy(arguments: [ + "from": .string("tmp:a.txt"), + "to": .string("tmp:b.txt"), + "overwrite": .bool(true), + ], context: context) + let replaced = try fs.read(arguments: ["path": .string("tmp:b.txt")], context: context) + #expect(try requireObject(replaced).string("text") == "source") +} + +@Test func fileSystemMoveOntoADirectoryAlsoRequiresRecursive() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: ["path": .string("tmp:a.txt"), "data": .string("source")], context: context) + _ = try fs.mkdir(arguments: ["path": .string("tmp:dir")], context: context) + _ = try fs.write(arguments: ["path": .string("tmp:dir/kept.txt"), "data": .string("kept")], context: context) + + do { + _ = try fs.move(arguments: [ + "from": .string("tmp:a.txt"), + "to": .string("tmp:dir"), + "overwrite": .bool(true), + ], context: context) + Issue.record("Expected fs.move onto a directory to require recursive=true") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + #expect(try fs.exists(arguments: ["path": .string("tmp:dir/kept.txt")], context: context).boolValue == true) + + _ = try fs.move(arguments: [ + "from": .string("tmp:a.txt"), + "to": .string("tmp:dir"), + "overwrite": .bool(true), + "recursive": .bool(true), + ], context: context) + #expect(try fs.exists(arguments: ["path": .string("tmp:dir/kept.txt")], context: context).boolValue == false) +} + +// MARK: - Size and decoding limits + +@Test func fileSystemReadRefusesFilesOverTheLimit() throws { + let fs = FileSystemBridge(limits: FileSystemLimits(maxReadBytes: 64, maxWriteBytes: 1_024)) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + _ = try fs.write(arguments: [ + "path": .string("tmp:big.txt"), + "data": .string(String(repeating: "x", count: 512)), + ], context: context) + + do { + _ = try fs.read(arguments: ["path": .string("tmp:big.txt")], context: context) + Issue.record("Expected fs.read to refuse a file over the read limit") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } +} + +@Test func fileSystemWriteRefusesPayloadsOverTheLimit() throws { + let fs = FileSystemBridge(limits: FileSystemLimits(maxReadBytes: 1_024, maxWriteBytes: 16)) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + do { + _ = try fs.write(arguments: [ + "path": .string("tmp:nested/big.txt"), + "data": .string(String(repeating: "x", count: 512)), + ], context: context) + Issue.record("Expected fs.write to refuse a payload over the write limit") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + + // A refused write leaves no directory behind. + #expect(try fs.exists(arguments: ["path": .string("tmp:nested")], context: context).boolValue == false) +} + +@Test func fileSystemReadReportsUndecodableUTF8InsteadOfEmptyText() throws { + let fs = FileSystemBridge() + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + // 0xFF is never valid UTF-8. Returning "" here told the script the file was + // empty; it must be an error the script can repair with base64. + let binary = Data([0xFF, 0xFE, 0x00, 0x01]) + _ = try fs.write(arguments: [ + "path": .string("tmp:binary.bin"), + "data": .string(binary.base64EncodedString()), + "encoding": .string("base64"), + ], context: context) + + do { + _ = try fs.read(arguments: ["path": .string("tmp:binary.bin")], context: context) + Issue.record("Expected fs.read to fail on undecodable UTF-8") + } catch { + #expect(requireBridgeErrorCode(error) == "INVALID_ARGUMENTS") + } + + let base64 = try fs.read(arguments: [ + "path": .string("tmp:binary.bin"), + "encoding": .string("base64"), + ], context: context) + #expect(try requireObject(base64).string("base64") == binary.base64EncodedString()) +} diff --git a/Tests/CodeModeTests/capability-metadata-golden.json b/Tests/CodeModeTests/capability-metadata-golden.json index 8e51716..ea1c45a 100644 --- a/Tests/CodeModeTests/capability-metadata-golden.json +++ b/Tests/CodeModeTests/capability-metadata-golden.json @@ -1852,10 +1852,14 @@ }, "argumentHints" : { "from" : "Source sandbox path.", - "to" : "Destination sandbox path." + "overwrite" : "Required as true to replace an existing destination; default false.", + "recursive" : "Required as true alongside overwrite when the existing destination is a directory.", + "to" : "Destination sandbox path. Must name a path inside a root, not the root itself." }, "argumentTypes" : { "from" : "string", + "overwrite" : "bool", + "recursive" : "bool", "to" : "string" }, "example" : "await apple.fs.copy({ from: 'tmp:a.txt', to: 'tmp:b.txt' })", @@ -1865,7 +1869,8 @@ "fs.promises.copyFile" ], "optionalArguments" : [ - + "overwrite", + "recursive" ], "requiredArguments" : [ "from", @@ -1875,7 +1880,7 @@ ], "resultSummary" : "Object with from\/to resolved paths.", - "summary" : "Copy file\/directory within allowed sandbox roots.", + "summary" : "Copy file\/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination.", "tags" : [ "filesystem", "io", @@ -2028,10 +2033,14 @@ }, "argumentHints" : { "from" : "Source sandbox path.", - "to" : "Destination sandbox path." + "overwrite" : "Required as true to replace an existing destination; default false.", + "recursive" : "Required as true alongside overwrite when the existing destination is a directory.", + "to" : "Destination sandbox path. Must name a path inside a root, not the root itself." }, "argumentTypes" : { "from" : "string", + "overwrite" : "bool", + "recursive" : "bool", "to" : "string" }, "example" : "await apple.fs.move({ from: 'tmp:a.txt', to: 'tmp:b.txt' })", @@ -2041,7 +2050,8 @@ "fs.promises.rename" ], "optionalArguments" : [ - + "overwrite", + "recursive" ], "requiredArguments" : [ "from", @@ -2051,7 +2061,7 @@ ], "resultSummary" : "Object with from\/to resolved paths.", - "summary" : "Move file\/directory within allowed sandbox roots.", + "summary" : "Move file\/directory within allowed sandbox roots. Fails rather than clobbering an existing destination unless overwrite is set, and never accepts a sandbox root as the destination.", "tags" : [ "filesystem", "io", @@ -2061,10 +2071,14 @@ }, { "allowedStringValues" : { - + "encoding" : [ + "utf8", + "base64", + "utf-8" + ] }, "argumentHints" : { - "encoding" : "utf8 (default) or base64.", + "encoding" : "utf8 (default) or base64. Binary files must use base64; a utf8 read of undecodable bytes fails rather than returning empty text.", "path" : "File path using allowed root prefix." }, "argumentTypes" : { @@ -2131,7 +2145,11 @@ }, { "allowedStringValues" : { - + "encoding" : [ + "utf8", + "base64", + "utf-8" + ] }, "argumentHints" : { "data" : "UTF-8 text or base64 string depending on encoding.", From aa96ec07227c334720e464ae9875c68fca2dd86c Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 19:54:47 -0700 Subject: [PATCH 04/18] Strip ambient authority from network.fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shipping default was `URLSession.shared`, which reads `HTTPCookieStorage.shared` and `URLCredentialStorage.shared`. Every script request therefore rode whatever the host app was already logged in to — authenticated session-riding against any origin with a live cookie — and a `Set-Cookie` in a script-fetched response poisoned the app's cookie jar. The tests all passed an `.ephemeral` session, so the shipping default was the one configuration never exercised. `network.fetch` now defaults to `NetworkBridge.isolatedSession`: ephemeral, with no cookie storage, no credential storage, no shared cache, and cookies disabled outright. Per-request cookie handling is disabled too, so a host that supplies its own session still does not attach its jar. Caller-supplied `Cookie`, `Authorization`, and `Proxy-*` headers were copied to the wire verbatim; they are now refused with `NETWORK_POLICY_VIOLATION` and audited. `NetworkAccessPolicy.allowsCredentialHeaders` re-enables them for hosts whose scripts legitimately call an authenticated API, and `.permissive` sets it, keeping that opt-out a faithful "old behavior" switch. Review finding P0 #5. --- README.md | 6 ++ .../CapabilityRegistrations+Core.swift | 4 +- Sources/CodeMode/Bridges/NetworkBridge.swift | 39 ++++++++-- .../Security/NetworkAccessPolicy.swift | 46 ++++++++++-- Tests/CodeModeTests/NetworkBridgeTests.swift | 71 +++++++++++++++++++ .../capability-metadata-golden.json | 4 +- 6 files changed, 157 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 453d109..d8a45a0 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,12 @@ let tools = CodeModeAgentTools( - `allowedHosts` restricts fetch to an explicit list (entries match the host and its subdomains, and deliberately allowlisted private hosts such as `localhost` are honored); `blockedHosts` refuses specific hosts; `NetworkAccessPolicy.permissive` restores unrestricted behavior. - Matching is by URL host only; DNS resolution is not performed, so a public hostname that resolves to a private address is not detected. Hosts that need stricter guarantees should set `allowedHosts`. +Script traffic also carries no ambient authority: + +- `network.fetch` runs on an isolated ephemeral `URLSession` with no cookie storage, no credential storage, and no shared cache — not `URLSession.shared`, which would let a script ride whatever the host app is already logged in to and let a `Set-Cookie` poison the app's cookie jar. +- Script-supplied `Cookie`, `Authorization`, and `Proxy-*` request headers are refused. Set `allowsCredentialHeaders: true` when scripts legitimately call an authenticated API. +- A host that passes its own `session` to `NetworkBridge` takes responsibility for this; requests still disable per-request cookie handling. + ```swift let tools = CodeModeAgentTools( config: CodeModeConfiguration( diff --git a/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift b/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift index 706b4af..1c210e3 100644 --- a/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift +++ b/Sources/CodeMode/Bridges/CapabilityRegistrations+Core.swift @@ -14,7 +14,7 @@ extension DefaultCapabilityRegistrationBuilder { descriptor: .init( id: .networkFetch, title: "Fetch HTTP resource", - summary: "Perform HTTP(S) requests through URLSession via a fetch-compatible API.", + summary: "Perform HTTP(S) requests through an isolated URLSession via a fetch-compatible API. Requests carry no app cookies or stored credentials, and credential headers (Cookie, Authorization, Proxy-*) are refused unless the host app permits them.", tags: ["network", "http", "fetch"], example: "await fetch('https://api.example.com/data').then(r => r.json())", requiredArguments: ["url"], @@ -29,7 +29,7 @@ extension DefaultCapabilityRegistrationBuilder { argumentHints: [ "url": "Absolute HTTP(S) URL string.", "options.method": "HTTP method; defaults to GET.", - "options.headers": "Object of header key/value string pairs.", + "options.headers": "Object of header key/value string pairs. Cookie, Authorization, and Proxy-* are refused unless the host app permits credential headers.", "options.body": "UTF-8 request body string.", "options.bodyBase64": "Base64-encoded request body. Mutually exclusive with options.body.", "options.timeoutMs": "Request and bridge wait timeout in milliseconds; defaults to 30000.", diff --git a/Sources/CodeMode/Bridges/NetworkBridge.swift b/Sources/CodeMode/Bridges/NetworkBridge.swift index ef33703..01234a9 100644 --- a/Sources/CodeMode/Bridges/NetworkBridge.swift +++ b/Sources/CodeMode/Bridges/NetworkBridge.swift @@ -6,7 +6,27 @@ public final class NetworkBridge: @unchecked Sendable { private let session: URLSession private let policy: NetworkAccessPolicy - public init(session: URLSession = .shared, policy: NetworkAccessPolicy = .standard) { + /// The session `network.fetch` uses unless the host supplies its own. + /// + /// Deliberately *not* `URLSession.shared`: that session reads + /// `HTTPCookieStorage.shared` and `URLCredentialStorage.shared`, so every + /// script request would ride whatever the host app is already logged in to, + /// and a `Set-Cookie` in a script-fetched response would poison the app's + /// cookie jar. The sandbox's premise is that script traffic carries no + /// ambient authority, so this session has no cookie jar, no credential + /// store, and no shared cache. + public static let isolatedSession: URLSession = { + let configuration = URLSessionConfiguration.ephemeral + configuration.httpCookieStorage = nil + configuration.urlCredentialStorage = nil + configuration.urlCache = nil + configuration.httpShouldSetCookies = false + configuration.httpCookieAcceptPolicy = .never + configuration.requestCachePolicy = .reloadIgnoringLocalCacheData + return URLSession(configuration: configuration) + }() + + public init(session: URLSession = NetworkBridge.isolatedSession, policy: NetworkAccessPolicy = .standard) { self.session = session self.policy = policy } @@ -43,12 +63,23 @@ public final class NetworkBridge: @unchecked Sendable { var request = URLRequest(url: url) request.httpMethod = options.string("method")?.uppercased() ?? "GET" request.timeoutInterval = TimeInterval(timeoutMs) / 1_000 + // Belt and braces alongside the isolated session: even a host-supplied + // session must not attach its cookie jar to script traffic. + request.httpShouldHandleCookies = false if let headers = options.object("headers") { - for (key, value) in headers { - if let string = value.stringValue { - request.setValue(string, forHTTPHeaderField: key) + for key in headers.keys.sorted() { + guard let string = headers[key]?.stringValue else { + continue } + if let reason = policy.headerViolationReason(for: key) { + context.auditLogger.log(AuditEvent( + capability: CapabilityID.networkFetch.rawValue, + message: "denied \(urlString): \(reason)" + )) + throw BridgeError.networkPolicyViolation(reason) + } + request.setValue(string, forHTTPHeaderField: key) } } diff --git a/Sources/CodeMode/Security/NetworkAccessPolicy.swift b/Sources/CodeMode/Security/NetworkAccessPolicy.swift index 3dd10f8..f391301 100644 --- a/Sources/CodeMode/Security/NetworkAccessPolicy.swift +++ b/Sources/CodeMode/Security/NetworkAccessPolicy.swift @@ -31,28 +31,64 @@ public struct NetworkAccessPolicy: Sendable, Equatable { /// that exceed this are cancelled mid-transfer. public var maxResponseBytes: Int + /// Whether script-supplied credential headers reach the wire. + /// + /// `Cookie`, `Authorization`, and the `Proxy-*` pair are how a request + /// carries authority. Script-authored code should not be minting them + /// against arbitrary origins, so they are refused by default. Hosts whose + /// scripts legitimately call an authenticated API — a bearer token the + /// script fetched from the keychain, say — set this to true. + public var allowsCredentialHeaders: Bool + + /// Request headers refused when `allowsCredentialHeaders` is false. + /// Matched case-insensitively; `proxy-` is matched as a prefix. + static let credentialHeaderNames: Set = [ + "cookie", + "cookie2", + "authorization", + ] + public init( allowedHosts: [String]? = nil, blockedHosts: [String] = [], blocksPrivateNetworks: Bool = true, - maxResponseBytes: Int = 10_485_760 + maxResponseBytes: Int = 10_485_760, + allowsCredentialHeaders: Bool = false ) { self.allowedHosts = allowedHosts self.blockedHosts = blockedHosts self.blocksPrivateNetworks = blocksPrivateNetworks self.maxResponseBytes = maxResponseBytes + self.allowsCredentialHeaders = allowsCredentialHeaders } - /// Secure default: private networks blocked, 10 MB response cap. + /// Secure default: private networks blocked, 10 MB response cap, no + /// script-supplied credential headers. public static let standard = NetworkAccessPolicy() - /// No destination restrictions and no response size cap. Matches the - /// pre-policy behavior of `network.fetch`; opt in deliberately. + /// No destination restrictions, no response size cap, and script-supplied + /// credential headers permitted. Matches the pre-policy behavior of + /// `network.fetch`; opt in deliberately. public static let permissive = NetworkAccessPolicy( blocksPrivateNetworks: false, - maxResponseBytes: .max + maxResponseBytes: .max, + allowsCredentialHeaders: true ) + /// Returns a reason the header is refused, or nil when it may be sent. + func headerViolationReason(for name: String) -> String? { + guard allowsCredentialHeaders == false else { + return nil + } + + let normalized = name.lowercased() + guard Self.credentialHeaderNames.contains(normalized) || normalized.hasPrefix("proxy-") else { + return nil + } + + return "the \"\(name)\" request header carries authority and is refused by the host app's network access policy" + } + /// Returns a human-readable reason the URL is refused, or nil when allowed. func violationReason(for url: URL) -> String? { guard let rawHost = url.host, rawHost.isEmpty == false else { diff --git a/Tests/CodeModeTests/NetworkBridgeTests.swift b/Tests/CodeModeTests/NetworkBridgeTests.swift index 5e444ae..ec72f4e 100644 --- a/Tests/CodeModeTests/NetworkBridgeTests.swift +++ b/Tests/CodeModeTests/NetworkBridgeTests.swift @@ -227,3 +227,74 @@ private final class StubHTTPURLProtocol: URLProtocol { #expect(result.string("stringValue") == "https://example.com/path?q=1") #expect(result.string("methodValue") == "https://example.com/path?q=1") } + +// MARK: - Ambient authority + +@Test func networkBridgeDefaultSessionCarriesNoAmbientCredentials() { + // The shipping default used to be `URLSession.shared`, which reads + // HTTPCookieStorage.shared and URLCredentialStorage.shared — so a script got + // authenticated session-riding against every origin the app is logged in to, + // and a Set-Cookie in a script-fetched response poisoned the app's jar. + let configuration = NetworkBridge.isolatedSession.configuration + #expect(configuration.httpCookieStorage == nil) + #expect(configuration.urlCredentialStorage == nil) + #expect(configuration.urlCache == nil) + #expect(configuration.httpShouldSetCookies == false) + #expect(configuration.httpCookieAcceptPolicy == .never) + #expect(NetworkBridge.isolatedSession !== URLSession.shared) +} + +@Test func networkFetchRefusesCredentialHeadersByDefault() throws { + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [StubHTTPURLProtocol.self] + let session = URLSession(configuration: configuration) + + let bridge = NetworkBridge(session: session) + let (context, sandbox) = try makeInvocationContext() + defer { + cleanup(sandbox) + session.invalidateAndCancel() + } + + for header in ["Cookie", "cookie", "Authorization", "Proxy-Authorization", "proxy-authenticate"] { + do { + _ = try bridge.fetch(arguments: [ + "url": .string("https://unit.test/endpoint"), + "options": .object(["headers": .object([header: .string("secret")])]), + ], context: context) + Issue.record("Expected \(header) to be refused") + } catch { + #expect(requireBridgeErrorCode(error) == "NETWORK_POLICY_VIOLATION") + } + } + + // Ordinary headers are unaffected. + let ok = try bridge.fetch(arguments: [ + "url": .string("https://unit.test/endpoint"), + "options": .object(["headers": .object(["X-Unit": .string("plain")])]), + ], context: context) + #expect(try requireObject(ok).bool("ok") == true) +} + +@Test func networkFetchAllowsCredentialHeadersWhenHostOptsIn() throws { + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [StubHTTPURLProtocol.self] + let session = URLSession(configuration: configuration) + + let bridge = NetworkBridge( + session: session, + policy: NetworkAccessPolicy(allowsCredentialHeaders: true) + ) + let (context, sandbox) = try makeInvocationContext() + defer { + cleanup(sandbox) + session.invalidateAndCancel() + } + + let result = try bridge.fetch(arguments: [ + "url": .string("https://unit.test/endpoint"), + "options": .object(["headers": .object(["Authorization": .string("Bearer t")])]), + ], context: context) + #expect(try requireObject(result).bool("ok") == true) + #expect(NetworkAccessPolicy.permissive.allowsCredentialHeaders) +} diff --git a/Tests/CodeModeTests/capability-metadata-golden.json b/Tests/CodeModeTests/capability-metadata-golden.json index ea1c45a..8569b96 100644 --- a/Tests/CodeModeTests/capability-metadata-golden.json +++ b/Tests/CodeModeTests/capability-metadata-golden.json @@ -3283,7 +3283,7 @@ "argumentHints" : { "options.body" : "UTF-8 request body string.", "options.bodyBase64" : "Base64-encoded request body. Mutually exclusive with options.body.", - "options.headers" : "Object of header key\/value string pairs.", + "options.headers" : "Object of header key\/value string pairs. Cookie, Authorization, and Proxy-* are refused unless the host app permits credential headers.", "options.method" : "HTTP method; defaults to GET.", "options.responseEncoding" : "text (default) or base64.", "options.timeoutMs" : "Request and bridge wait timeout in milliseconds; defaults to 30000.", @@ -3318,7 +3318,7 @@ ], "resultSummary" : "Object with ok\/status\/statusText\/headers\/bodyText or bodyBase64.", - "summary" : "Perform HTTP(S) requests through URLSession via a fetch-compatible API.", + "summary" : "Perform HTTP(S) requests through an isolated URLSession via a fetch-compatible API. Requests carry no app cookies or stored credentials, and credential headers (Cookie, Authorization, Proxy-*) are refused unless the host app permits them.", "tags" : [ "network", "http", From d8aa9da53e2dad7696202479709aba5eac39152e Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 19:58:26 -0700 Subject: [PATCH 05/18] Bound what a single execution can accumulate in memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing capped output. The whole result was stringified, copied into a Swift String, and re-decoded; every console.log was retained and then copied again into each CodeModeToolError; the events AsyncStream used the default unbounded buffering, so a chatty script whose events the host was not draining buffered without limit. One script could jetsam an iOS host. `CodeModeConfiguration.executionLimits` bounds all of it, and every limit degrades rather than fails — for an LLM consumer a truncated result with a diagnostic beats an out-of-memory kill: - Result size is measured *inside* JS, so an oversized value is never brought across. Over the limit, the output is replaced by a still-parseable descriptor (`__codeModeTruncated`, character count, limit, preview) plus a RESULT_TRUNCATED diagnostic suggesting fs.write + return the path. - Retained logs are capped at the first N with a one-time LOG_LIMIT_REACHED notice; streaming stays live past the cap since it is the retained copy that costs. Over-long messages are elided in the middle, keeping head and tail. - Diagnostics and permission events are capped. - The events stream uses `.bufferingNewest(n)`. Two watchdog findings in the same code path: - The serialization re-arm used `max(timeoutMs, 1000)`, so a 60s execution bought another 60s for JSON.stringify. It is `min` against a fixed 1s budget now, which is what "bounded budget" meant. - `watchdog.termination` was never consulted after the re-arm, so a runaway getter surfaced as `INVALID_RESULT: must be JSON-serializable` and steered the model to the wrong fix. It now reports EXECUTION_TIMEOUT/CANCELLED naming the serialization phase. Review finding P0 #6 and the first two P2 items. --- Sources/CodeMode/API/BridgeModels.swift | 4 + Sources/CodeMode/Runtime/BridgeRuntime.swift | 108 +++++++++++++-- .../CodeMode/Runtime/ExecutionLimits.swift | 59 ++++++++ .../CodeMode/Runtime/ExecutionSupport.swift | 66 ++++++++- .../CodeModeTests/ExecutionLimitsTests.swift | 131 ++++++++++++++++++ .../ExecutionWatchdogTests.swift | 26 +++- Tests/CodeModeTests/TestSupport.swift | 2 + 7 files changed, 380 insertions(+), 16 deletions(-) create mode 100644 Sources/CodeMode/Runtime/ExecutionLimits.swift create mode 100644 Tests/CodeModeTests/ExecutionLimitsTests.swift diff --git a/Sources/CodeMode/API/BridgeModels.swift b/Sources/CodeMode/API/BridgeModels.swift index b5c6caa..8c2bca5 100644 --- a/Sources/CodeMode/API/BridgeModels.swift +++ b/Sources/CodeMode/API/BridgeModels.swift @@ -9,6 +9,8 @@ public struct CodeModeConfiguration: Sendable { public var fileSystem: any CodeModeFileSystem /// Byte ceilings applied by the filesystem bridge to `fs.read` / `fs.write`. public var fileSystemLimits: FileSystemLimits + /// Bounds on retained result/log/event volume for a single execution. + public var executionLimits: ExecutionLimits public var artifactStore: any ArtifactStore public var permissionBroker: any PermissionBroker public var auditLogger: any AuditLogger @@ -33,6 +35,7 @@ public struct CodeModeConfiguration: Sendable { capabilityGrant: CapabilityGrant = .unrestricted, fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), fileSystemLimits: FileSystemLimits = .standard, + executionLimits: ExecutionLimits = .standard, artifactStore: any ArtifactStore = InMemoryArtifactStore(), permissionBroker: any PermissionBroker = SystemPermissionBroker(), auditLogger: any AuditLogger = SyncAuditLogger(), @@ -56,6 +59,7 @@ public struct CodeModeConfiguration: Sendable { self.capabilityGrant = capabilityGrant self.fileSystem = fileSystem self.fileSystemLimits = fileSystemLimits + self.executionLimits = executionLimits self.artifactStore = artifactStore self.permissionBroker = permissionBroker self.auditLogger = auditLogger diff --git a/Sources/CodeMode/Runtime/BridgeRuntime.swift b/Sources/CodeMode/Runtime/BridgeRuntime.swift index 5e2ad3a..b7a1106 100644 --- a/Sources/CodeMode/Runtime/BridgeRuntime.swift +++ b/Sources/CodeMode/Runtime/BridgeRuntime.swift @@ -18,6 +18,11 @@ final class BridgeRuntime: @unchecked Sendable { var suggestions: [String] } + /// Ceiling on the re-armed watchdog budget for result serialization. Bounded + /// independently of the execution timeout so a long-running script cannot buy + /// an equally long serialization window. + private static let serializationBudgetMs = 1_000 + private let registry: CapabilityRegistry private let catalog: BridgeCatalog private let config: CodeModeConfiguration @@ -49,7 +54,7 @@ final class BridgeRuntime: @unchecked Sendable { ) } - let transcript = ExecutionTranscript() + let transcript = ExecutionTranscript(limits: config.executionLimits) let cancellationController = ExecutionCancellationController() let invocationContext = BridgeInvocationContext( executionContext: .init(), @@ -104,10 +109,16 @@ final class BridgeRuntime: @unchecked Sendable { func makeExecutionCall(_ request: JavaScriptExecutionRequest) -> JavaScriptExecutionCall { let cancellationController = ExecutionCancellationController() let continuationBox = LockedBox.Continuation?>(nil) - let events = AsyncStream { continuation in + // Bounded: the default policy buffers without limit, so a chatty script + // whose events the host is not draining grows the buffer until the app + // dies. Dropping the oldest keeps the terminal events — which is what + // the host actually needs — and the full transcript is on the result. + let events = AsyncStream( + bufferingPolicy: .bufferingNewest(config.executionLimits.maxBufferedEvents) + ) { continuation in continuationBox.set(continuation) } - let transcript = ExecutionTranscript { event in + let transcript = ExecutionTranscript(limits: config.executionLimits) { event in continuationBox.get()?.yield(event) } @@ -503,12 +514,18 @@ final class BridgeRuntime: @unchecked Sendable { // Give result serialization its own bounded budget so a script that // settled near the deadline still serializes, while a runaway getter or - // toJSON is terminated instead of hanging the execution thread. - watchdog.rearm(timeoutMs: max(timeoutMs, 1_000)) + // toJSON is terminated instead of hanging the execution thread. `min`, + // not `max`: the point is a *bounded* budget, and `max` handed a 60s + // execution another 60s just to run JSON.stringify. + watchdog.rearm(timeoutMs: min(timeoutMs, Self.serializationBudgetMs)) switch settlement { case .fulfilled: - let output = try decodeOutput(from: context) + let output = try decodeOutput( + from: context, + watchdog: watchdog, + invocationContext: invocationContext + ) if output == nil { invocationContext.recordDiagnostic(Self.noReturnValueDiagnostic(for: code)) } @@ -676,12 +693,14 @@ final class BridgeRuntime: @unchecked Sendable { cancelMessage: "Search cancelled" ) - watchdog.rearm(timeoutMs: max(timeoutMs, 1_000)) + watchdog.rearm(timeoutMs: min(timeoutMs, Self.serializationBudgetMs)) switch settlement { case .fulfilled: return try decodeOutput( from: context, + watchdog: watchdog, + invocationContext: invocationContext, errorCode: "INVALID_SEARCH_RESULT", errorMessagePrefix: "Search result must be JSON-serializable" ) @@ -693,6 +712,8 @@ final class BridgeRuntime: @unchecked Sendable { private func decodeOutput( from context: JSContext, + watchdog: ExecutionWatchdog? = nil, + invocationContext: BridgeInvocationContext? = nil, errorCode: String = "INVALID_RESULT", errorMessagePrefix: String = "Execution result must be JSON-serializable" ) throws -> JSONValue? { @@ -700,25 +721,75 @@ final class BridgeRuntime: @unchecked Sendable { return nil } + let limits = config.executionLimits + // Size is measured and the result is replaced *inside* JS: bringing a + // gigabyte of JSON across into a Swift String and then rejecting it has + // already spent the memory the cap exists to protect. The replacement is + // still valid JSON, so the model gets a parseable value plus a + // diagnostic rather than a truncated fragment it cannot read. guard let serialization = context.evaluateScript( """ (function(){ try { - return { ok: true, json: JSON.stringify(globalThis.__codemode.result) }; + var json = JSON.stringify(globalThis.__codemode.result); + if (typeof json !== 'string') { + return { ok: true, json: json }; + } + if (json.length > \(limits.maxResultCharacters)) { + return { + ok: true, + truncated: true, + size: json.length, + json: JSON.stringify({ + __codeModeTruncated: true, + characterCount: json.length, + limit: \(limits.maxResultCharacters), + preview: json.slice(0, \(limits.truncatedResultPreviewCharacters)) + }) + }; + } + return { ok: true, json: json }; } catch (error) { return { ok: false, message: String(error) }; } })() """ ) else { + // A nil evaluation here after the watchdog fired is a termination, + // not a serialization problem. Reporting `INVALID_RESULT: must be + // JSON-serializable` for a runaway getter steered the model to fix + // the wrong thing. + if let termination = watchdog?.termination { + throw terminationError(termination, invocationContext: invocationContext, during: "result serialization") + } throw CodeModeToolError(code: errorCode, message: errorMessagePrefix) } + if let termination = watchdog?.termination { + throw terminationError(termination, invocationContext: invocationContext, during: "result serialization") + } + if serialization.forProperty("ok")?.toBool() == false { let message = serialization.forProperty("message")?.toString() ?? errorMessagePrefix throw CodeModeToolError(code: errorCode, message: "\(errorMessagePrefix): \(message)") } + if serialization.forProperty("truncated")?.toBool() == true { + let size = serialization.forProperty("size")?.toNumber()?.intValue ?? limits.maxResultCharacters + invocationContext?.recordDiagnostic( + ToolDiagnostic( + severity: .warning, + code: "RESULT_TRUNCATED", + message: "The result serialized to \(size) characters, over the \(limits.maxResultCharacters)-character limit. It was replaced with a descriptor object carrying a \(limits.truncatedResultPreviewCharacters)-character preview.", + category: "execution", + suggestions: [ + "Return a summary, a count, or a slice instead of the whole collection.", + "Write large payloads to the sandbox with apple.fs.write and return the path.", + ] + ) + ) + } + guard let jsonValue = serialization.forProperty("json"), jsonValue.isUndefined == false, jsonValue.isNull == false, let serialized = jsonValue.toString() else { @@ -732,6 +803,27 @@ final class BridgeRuntime: @unchecked Sendable { return try JSONDecoder.codeModeBridge.decode(JSONValue.self, from: data) } + /// Reports a watchdog termination that happened outside the script body — so + /// far, during result serialization — as the timeout or cancellation it + /// actually is. + private func terminationError( + _ termination: ExecutionWatchdog.Termination, + invocationContext: BridgeInvocationContext?, + during phase: String + ) -> CodeModeToolError { + let (code, message): (String, String) = switch termination { + case .timedOut: + ("EXECUTION_TIMEOUT", "Execution timed out during \(phase). A getter or toJSON on the returned value did not finish within the serialization budget.") + case .cancelled: + ("CANCELLED", "Execution cancelled during \(phase)") + } + + guard let invocationContext else { + return CodeModeToolError(code: code, message: message) + } + return toolError(code: code, message: message, transcript: invocationContext) + } + private func bridgeFailurePayload( for error: BridgeError, capability: CapabilityID?, diff --git a/Sources/CodeMode/Runtime/ExecutionLimits.swift b/Sources/CodeMode/Runtime/ExecutionLimits.swift new file mode 100644 index 0000000..3a59c7e --- /dev/null +++ b/Sources/CodeMode/Runtime/ExecutionLimits.swift @@ -0,0 +1,59 @@ +import Foundation + +/// Bounds on how much an execution may accumulate. +/// +/// Nothing about a script's *output* was previously capped: the whole result was +/// stringified, copied into a Swift `String`, and re-decoded into a `JSONValue`; +/// every `console.log` was retained and then copied again into any +/// `CodeModeToolError`; and the events `AsyncStream` used unbounded buffering, so +/// a chatty script that the host was not draining buffered without limit. A +/// single script could jetsam an iOS host. +/// +/// For an LLM consumer, a truncated result carrying a diagnostic is strictly +/// better than an out-of-memory kill, so every limit here degrades rather than +/// fails. +public struct ExecutionLimits: Sendable, Equatable { + /// Maximum size of the serialized result, in JSON characters. A larger + /// result is replaced by a descriptor object carrying a prefix of the JSON. + public var maxResultCharacters: Int + + /// How much of the oversized JSON to include in that descriptor. + public var truncatedResultPreviewCharacters: Int + + /// Maximum retained `console.log` entries. + public var maxLogEntries: Int + + /// Maximum characters retained per log message; longer messages are elided + /// in the middle, which keeps both the prefix and the tail readable. + public var maxLogMessageCharacters: Int + + /// Maximum retained diagnostics. + public var maxDiagnostics: Int + + /// Maximum retained permission events. + public var maxPermissionEvents: Int + + /// Depth of the events `AsyncStream` buffer. A host that stops draining + /// drops the oldest events rather than growing without bound. + public var maxBufferedEvents: Int + + public init( + maxResultCharacters: Int = 1_000_000, + truncatedResultPreviewCharacters: Int = 4_000, + maxLogEntries: Int = 1_000, + maxLogMessageCharacters: Int = 8_000, + maxDiagnostics: Int = 200, + maxPermissionEvents: Int = 200, + maxBufferedEvents: Int = 1_000 + ) { + self.maxResultCharacters = maxResultCharacters + self.truncatedResultPreviewCharacters = truncatedResultPreviewCharacters + self.maxLogEntries = maxLogEntries + self.maxLogMessageCharacters = maxLogMessageCharacters + self.maxDiagnostics = maxDiagnostics + self.maxPermissionEvents = maxPermissionEvents + self.maxBufferedEvents = maxBufferedEvents + } + + public static let standard = ExecutionLimits() +} diff --git a/Sources/CodeMode/Runtime/ExecutionSupport.swift b/Sources/CodeMode/Runtime/ExecutionSupport.swift index 1b20c1f..6e49650 100644 --- a/Sources/CodeMode/Runtime/ExecutionSupport.swift +++ b/Sources/CodeMode/Runtime/ExecutionSupport.swift @@ -19,38 +19,90 @@ final class ExecutionCancellationController: @unchecked Sendable { final class ExecutionTranscript: @unchecked Sendable { private let lock = NSLock() + private let limits: ExecutionLimits private var logs: [ExecutionLog] = [] private var diagnostics: [ToolDiagnostic] = [] private var permissionEvents: [PermissionEvent] = [] + private var droppedLogs = 0 + private var noticedLogOverflow = false private let emitEvent: @Sendable (JavaScriptExecutionEvent) -> Void - init(emitEvent: @escaping @Sendable (JavaScriptExecutionEvent) -> Void = { _ in }) { + init( + limits: ExecutionLimits = .standard, + emitEvent: @escaping @Sendable (JavaScriptExecutionEvent) -> Void = { _ in } + ) { + self.limits = limits self.emitEvent = emitEvent } func record(log: ExecutionLog) { + var entry = log + entry.message = Self.elide(entry.message, to: limits.maxLogMessageCharacters) + + var overflowNotice: ToolDiagnostic? lock.lock() - logs.append(log) - lock.unlock() - emitEvent(.log(log)) + if logs.count >= limits.maxLogEntries { + // Keep the first N: the beginning of a runaway log is where the + // cause is, and the tail is the same line a million times. + droppedLogs += 1 + if noticedLogOverflow == false { + noticedLogOverflow = true + overflowNotice = ToolDiagnostic( + severity: .warning, + code: "LOG_LIMIT_REACHED", + message: "Kept the first \(limits.maxLogEntries) console.log entries; later entries are dropped.", + category: "execution", + suggestions: ["Log summaries rather than per-item lines, or return the data instead of logging it."] + ) + } + lock.unlock() + } else { + logs.append(entry) + lock.unlock() + // Streaming stays live even after the retained transcript is full; + // it is the retained copy — re-copied into every CodeModeToolError — + // that is the memory risk. + emitEvent(.log(entry)) + } + + if let overflowNotice { + record(diagnostic: overflowNotice) + } } func record(diagnostic: ToolDiagnostic) { lock.lock() - diagnostics.append(diagnostic) + let accepted = diagnostics.count < limits.maxDiagnostics + if accepted { + diagnostics.append(diagnostic) + } lock.unlock() - if diagnostic.severity != .error { + if accepted, diagnostic.severity != .error { emitEvent(.diagnostic(diagnostic)) } } func record(permissionEvent: PermissionEvent) { lock.lock() - permissionEvents.append(permissionEvent) + if permissionEvents.count < limits.maxPermissionEvents { + permissionEvents.append(permissionEvent) + } lock.unlock() } + /// Elides the middle of an over-long message, keeping both ends: a stack + /// trace's origin and its outcome are usually at opposite ends. + private static func elide(_ message: String, to limit: Int) -> String { + guard limit > 0, message.count > limit else { + return message + } + let keep = max(1, (limit - 1) / 2) + let head = message.prefix(keep) + let tail = message.suffix(keep) + return "\(head)… [\(message.count - 2 * keep) characters elided] …\(tail)" + } + func snapshot(output: JSONValue? = nil) -> JavaScriptExecutionResult { lock.lock() defer { lock.unlock() } diff --git a/Tests/CodeModeTests/ExecutionLimitsTests.swift b/Tests/CodeModeTests/ExecutionLimitsTests.swift new file mode 100644 index 0000000..7dae8ec --- /dev/null +++ b/Tests/CodeModeTests/ExecutionLimitsTests.swift @@ -0,0 +1,131 @@ +import Foundation +import Testing +@testable import CodeMode + +// Nothing bounded a script's output before: the whole result was stringified and +// copied into Swift, every console.log was retained and re-copied into each +// CodeModeToolError, and the events stream buffered without limit. One script +// could jetsam an iOS host. Every limit degrades with a diagnostic instead of +// failing, because for an LLM consumer a truncated result beats an OOM. + +@Test func oversizedResultsAreReplacedWithAParseableDescriptor() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxResultCharacters: 2_000, truncatedResultPreviewCharacters: 200) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "return Array.from({ length: 5000 }, (_, i) => 'entry-' + i);", + allowedCapabilities: [] + ) + ) + + #expect(observed.error == nil) + let output = try #require(observed.result?.output?.objectValue) + #expect(output.bool("__codeModeTruncated") == true) + #expect((output.int("characterCount") ?? 0) > 2_000) + #expect(output.int("limit") == 2_000) + #expect((output.string("preview")?.count ?? 0) <= 200) + + let diagnostics = try #require(observed.result?.diagnostics) + #expect(diagnostics.contains { $0.code == "RESULT_TRUNCATED" }) +} + +@Test func resultsWithinTheLimitAreUntouched() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxResultCharacters: 100_000) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest(code: "return { ok: true, items: [1, 2, 3] };", allowedCapabilities: []) + ) + + let output = try #require(observed.result?.output?.objectValue) + #expect(output.bool("ok") == true) + #expect(output.array("items")?.count == 3) + #expect(observed.result?.diagnostics.contains { $0.code == "RESULT_TRUNCATED" } == false) +} + +@Test func retainedLogsAreCappedWithANotice() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxLogEntries: 25) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "for (let i = 0; i < 500; i++) { console.log('line ' + i); } return 'done';", + allowedCapabilities: [] + ) + ) + + let logs = try #require(observed.result?.logs) + #expect(logs.count == 25) + // The first N are kept: the beginning of a runaway log is where the cause is. + #expect(logs.first?.message == "line 0") + #expect(observed.result?.diagnostics.contains { $0.code == "LOG_LIMIT_REACHED" } == true) +} + +@Test func longLogMessagesAreElidedInTheMiddle() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxLogMessageCharacters: 200) + ) + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "console.log('HEAD' + 'x'.repeat(100000) + 'TAIL'); return 'done';", + allowedCapabilities: [] + ) + ) + + let message = try #require(observed.result?.logs.first?.message) + #expect(message.count < 400) + #expect(message.hasPrefix("HEAD")) + #expect(message.hasSuffix("TAIL")) + #expect(message.contains("characters elided")) +} + +@Test func diagnosticsAndPermissionEventsAreCapped() { + let transcript = ExecutionTranscript( + limits: ExecutionLimits(maxDiagnostics: 3, maxPermissionEvents: 2) + ) + + for index in 0..<50 { + transcript.record(diagnostic: ToolDiagnostic(severity: .info, code: "D\(index)", message: "m")) + transcript.record(permissionEvent: PermissionEvent(permission: .contacts, status: .granted)) + } + + let snapshot = transcript.snapshot() + #expect(snapshot.diagnostics.count == 3) + #expect(snapshot.permissionEvents.count == 2) +} + +@Test func theEventStreamBufferIsBounded() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxLogEntries: 100_000, maxBufferedEvents: 16) + ) + defer { cleanup(sandbox) } + + // Never drained until the script has finished: with the default unbounded + // policy this buffers every event; bounded, it keeps only the newest. + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "for (let i = 0; i < 5000; i++) { console.log('line ' + i); } return 'done';", + allowedCapabilities: [] + ) + ) + _ = try await call.result + + var received = 0 + for await _ in call.events { + received += 1 + } + #expect(received <= 16) +} diff --git a/Tests/CodeModeTests/ExecutionWatchdogTests.swift b/Tests/CodeModeTests/ExecutionWatchdogTests.swift index 1e6012d..f562599 100644 --- a/Tests/CodeModeTests/ExecutionWatchdogTests.swift +++ b/Tests/CodeModeTests/ExecutionWatchdogTests.swift @@ -151,10 +151,34 @@ import Testing ) #expect(observed.result?.output == nil) - #expect(observed.error != nil) + // The watchdog fired, so this is a timeout — not the INVALID_RESULT + // "must be JSON-serializable" it used to report, which pointed the model at + // the wrong repair entirely. + #expect(observed.error?.code == "EXECUTION_TIMEOUT") #expect(Date().timeIntervalSince(started) < 5) } +@Test func serializationBudgetDoesNotScaleWithTheExecutionTimeout() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // A long execution budget used to buy an equally long serialization budget + // (`max`, not `min`), so a runaway getter after a 30s-budget script got + // another 30s. The serialization phase is bounded independently. + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "return { get trap() { while (true) {} } };", + allowedCapabilities: [], + timeoutMs: 30_000 + ) + ) + + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + #expect(Date().timeIntervalSince(started) < 10) +} + @Test func searchTerminatesCPUBoundInfiniteLoop() async throws { let (tools, sandbox) = try makeTools() defer { cleanup(sandbox) } diff --git a/Tests/CodeModeTests/TestSupport.swift b/Tests/CodeModeTests/TestSupport.swift index 8c31603..6973d2b 100644 --- a/Tests/CodeModeTests/TestSupport.swift +++ b/Tests/CodeModeTests/TestSupport.swift @@ -36,6 +36,7 @@ func cleanup(_ sandbox: TestSandbox) { func makeTools( permissionBroker: any PermissionBroker = NoopPermissionBroker(), capabilityGrant: CapabilityGrant = .unrestricted, + executionLimits: ExecutionLimits = .standard, fileSystem: any CodeModeFileSystem = LocalCodeModeFileSystem(), systemUIPresenter: any SystemUIPresenter = UnavailableSystemUIPresenter(), eventInbox: any CodeModeEventInbox = UnavailableCodeModeEventInbox(), @@ -62,6 +63,7 @@ func makeTools( pathPolicy: pathPolicy, capabilityGrant: capabilityGrant, fileSystem: fileSystem, + executionLimits: executionLimits, artifactStore: InMemoryArtifactStore(), permissionBroker: permissionBroker, auditLogger: SyncAuditLogger(), From cdc991acded120f761a6c03b82aef1b01e8e226b Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 20:01:29 -0700 Subject: [PATCH 06/18] Fix two threading bugs in the permission and EventKit paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Location.** `CLLocationManager` delivers delegate callbacks on the run loop of the thread that *created* it, and the bridge's GCD workers have none. The manager and delegate were constructed on the execution thread with only `requestWhenInUseAuthorization()` dispatched to main, so `locationManagerDidChangeAuthorization` never arrived: every first-time grant burned the full 10s wait and then reported from a status re-read that races the TCC write, so a user who tapped Allow could still get `notDetermined`. Creation, delegate assignment, and the request now all happen on main, the delegate holds the manager alive (CoreLocation does not retain its delegate, and the manager would otherwise die with the dispatched block), and the resolved status comes from the callback rather than a re-read. **Reminders.** `EventKitBridge.readReminders` wrote `var result` from EventKit's completion queue, read it after a `semaphore.wait` whose timeout result was discarded, and returned `.array([])` on timeout — a data race with the late callback, and a result the script could not tell apart from "no reminders". Both now go through one `CompletionWait` helper that either returns the value the callback delivered or throws `BridgeError.timeout`, keeping the box alive so a late callback writes somewhere harmless. The broker's `request*` methods are migrated to the same helper, which also collapses their duplicated availability-branch semaphore plumbing. Review findings P0 #7 and #8. --- Sources/CodeMode/Bridges/EventKitBridge.swift | 39 +++-- .../Security/SystemPermissionBroker.swift | 154 ++++++++++-------- Sources/CodeMode/Support/CompletionWait.swift | 55 +++++++ Tests/CodeModeTests/CompletionWaitTests.swift | 49 ++++++ 4 files changed, 215 insertions(+), 82 deletions(-) create mode 100644 Sources/CodeMode/Support/CompletionWait.swift create mode 100644 Tests/CodeModeTests/CompletionWaitTests.swift diff --git a/Sources/CodeMode/Bridges/EventKitBridge.swift b/Sources/CodeMode/Bridges/EventKitBridge.swift index 279c31b..cd7cdb8 100644 --- a/Sources/CodeMode/Bridges/EventKitBridge.swift +++ b/Sources/CodeMode/Bridges/EventKitBridge.swift @@ -5,6 +5,8 @@ import EventKit #endif public final class EventKitBridge: @unchecked Sendable { + static let reminderFetchTimeoutSeconds: TimeInterval = 15 + public init() {} public func readEvents(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { @@ -97,8 +99,6 @@ public final class EventKitBridge: @unchecked Sendable { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - var result: [JSONValue] = [] let includeCompleted = arguments.bool("includeCompleted") ?? false let start = isoDate(arguments.string("start")) let end = isoDate(arguments.string("end")) @@ -113,23 +113,30 @@ public final class EventKitBridge: @unchecked Sendable { let predicate = includeCompleted ? store.predicateForReminders(in: calendars) : store.predicateForIncompleteReminders(withDueDateStarting: start, ending: end, calendars: calendars) - store.fetchReminders(matching: predicate) { reminders in - let filtered = (reminders ?? []) - .filter { reminder in - guard includeCompleted else { return true } - guard let dueDate = reminder.dueDateComponents?.date else { - return start == nil && end == nil + // Previously a `var` written from EventKit's completion queue, read after + // a semaphore wait whose timeout result was discarded — a data race with + // the late callback, and an empty array on timeout that the script could + // not tell apart from "no reminders". + let result = try CompletionWait.value( + timeout: Self.reminderFetchTimeoutSeconds, + operationName: "reminders.read" + ) { complete in + store.fetchReminders(matching: predicate) { reminders in + let filtered = (reminders ?? []) + .filter { reminder in + guard includeCompleted else { return true } + guard let dueDate = reminder.dueDateComponents?.date else { + return start == nil && end == nil + } + if let start, dueDate < start { return false } + if let end, dueDate > end { return false } + return true } - if let start, dueDate < start { return false } - if let end, dueDate > end { return false } - return true - } - .prefix(limit) - result = filtered.map { Self.reminderJSON($0) } - semaphore.signal() + .prefix(limit) + complete(filtered.map { Self.reminderJSON($0) }) + } } - _ = semaphore.wait(timeout: .now() + 15) return .array(result) #else _ = arguments diff --git a/Sources/CodeMode/Security/SystemPermissionBroker.swift b/Sources/CodeMode/Security/SystemPermissionBroker.swift index 27f4451..b2cad2a 100644 --- a/Sources/CodeMode/Security/SystemPermissionBroker.swift +++ b/Sources/CodeMode/Security/SystemPermissionBroker.swift @@ -45,6 +45,12 @@ import HomeKit #endif public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable { + /// How long a `request(for:)` waits on the system's TCC dialog before giving + /// up and re-reading the status. This is a human-latency budget, and it is + /// short: a user who takes longer than this leaves the caller reading the + /// pre-prompt status. + static let permissionPromptTimeoutSeconds: TimeInterval = 10 + public init() {} public func status(for permission: PermissionKind) -> PermissionStatus { @@ -111,8 +117,15 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func locationStatus() -> PermissionStatus { #if canImport(CoreLocation) - let manager = CLLocationManager() - switch manager.authorizationStatus { + return Self.mapLocationAuthorization(CLLocationManager().authorizationStatus) + #else + return .unavailable + #endif + } + + #if canImport(CoreLocation) + static func mapLocationAuthorization(_ status: CLAuthorizationStatus) -> PermissionStatus { + switch status { case .authorizedWhenInUse, .authorizedAlways: return .granted case .denied: @@ -124,10 +137,8 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable @unknown default: return .unavailable } - #else - return .unavailable - #endif } + #endif private func contactsStatus() -> PermissionStatus { #if canImport(Contacts) @@ -419,24 +430,41 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestLocationPermission() -> PermissionStatus { #if canImport(CoreLocation) && os(iOS) - let manager = CLLocationManager() let delegate = LocationPermissionDelegate() - manager.delegate = delegate if Thread.isMainThread { + let manager = CLLocationManager() + manager.delegate = delegate + delegate.hold(manager) manager.requestWhenInUseAuthorization() + // Cannot block main for the callback; the caller sees the pre-prompt + // status and a later call observes the resolved one. return locationStatus() } - // async, not sync: execution runs on a background queue, and the host may be - // blocking the main thread waiting on the result — main.sync would deadlock. - // If main never runs the request, the delegate wait below times out instead. + // `CLLocationManager` delivers delegate callbacks on the run loop of the + // thread that *created* it, and the bridge's GCD workers have none — so a + // manager constructed here never called back, the wait below always burned + // its full 10s, and the result was reported from a status re-read that + // races the TCC write. Creating the manager, assigning the delegate, and + // requesting all happen on main. + // + // async, not sync: the host may be blocking main waiting on our result, so + // main.sync would deadlock. If main never runs this, the wait times out. DispatchQueue.main.async { + let manager = CLLocationManager() + manager.delegate = delegate + delegate.hold(manager) manager.requestWhenInUseAuthorization() } - _ = delegate.wait(timeout: 10) - return locationStatus() + guard delegate.wait(timeout: 10) == .success else { + return locationStatus() + } + + // Prefer the status the delegate callback carried over a fresh read, + // which can still observe the pre-prompt value. + return delegate.observedStatus().map(Self.mapLocationAuthorization) ?? locationStatus() #else return .unavailable #endif @@ -444,12 +472,10 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestContactsPermission() -> PermissionStatus { #if canImport(Contacts) - let semaphore = DispatchSemaphore(value: 0) let store = CNContactStore() - store.requestAccess(for: .contacts) { _, _ in - semaphore.signal() + CompletionWait.completion(timeout: Self.permissionPromptTimeoutSeconds) { complete in + store.requestAccess(for: .contacts) { _, _ in complete() } } - _ = semaphore.wait(timeout: .now() + 10) return contactsStatus() #else return .unavailable @@ -459,19 +485,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestCalendarPermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 17.0, macOS 14.0, *) { - store.requestFullAccessToEvents { _, _ in - semaphore.signal() - } - } else { - store.requestAccess(to: .event) { _, _ in - semaphore.signal() + CompletionWait.completion(timeout: Self.permissionPromptTimeoutSeconds) { complete in + if #available(iOS 17.0, macOS 14.0, *) { + store.requestFullAccessToEvents { _, _ in complete() } + } else { + store.requestAccess(to: .event) { _, _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return calendarReadStatus() #else return .unavailable @@ -481,19 +501,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestCalendarWritePermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 17.0, macOS 14.0, *) { - store.requestWriteOnlyAccessToEvents { _, _ in - semaphore.signal() - } - } else { - store.requestAccess(to: .event) { _, _ in - semaphore.signal() + CompletionWait.completion(timeout: Self.permissionPromptTimeoutSeconds) { complete in + if #available(iOS 17.0, macOS 14.0, *) { + store.requestWriteOnlyAccessToEvents { _, _ in complete() } + } else { + store.requestAccess(to: .event) { _, _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return calendarWriteStatus() #else return .unavailable @@ -503,19 +517,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestRemindersPermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 17.0, macOS 14.0, *) { - store.requestFullAccessToReminders { _, _ in - semaphore.signal() - } - } else { - store.requestAccess(to: .reminder) { _, _ in - semaphore.signal() + CompletionWait.completion(timeout: Self.permissionPromptTimeoutSeconds) { complete in + if #available(iOS 17.0, macOS 14.0, *) { + store.requestFullAccessToReminders { _, _ in complete() } + } else { + store.requestAccess(to: .reminder) { _, _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return remindersStatus() #else return .unavailable @@ -524,19 +532,13 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestPhotoLibraryPermission() -> PermissionStatus { #if canImport(Photos) - let semaphore = DispatchSemaphore(value: 0) - - if #available(iOS 14.0, macOS 11.0, *) { - PHPhotoLibrary.requestAuthorization(for: .readWrite) { _ in - semaphore.signal() - } - } else { - PHPhotoLibrary.requestAuthorization { _ in - semaphore.signal() + CompletionWait.completion(timeout: Self.permissionPromptTimeoutSeconds) { complete in + if #available(iOS 14.0, macOS 11.0, *) { + PHPhotoLibrary.requestAuthorization(for: .readWrite) { _ in complete() } + } else { + PHPhotoLibrary.requestAuthorization { _ in complete() } } } - - _ = semaphore.wait(timeout: .now() + 10) return photoLibraryStatus() #else return .unavailable @@ -545,13 +547,11 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestNotificationsPermission() -> PermissionStatus { #if canImport(UserNotifications) - let semaphore = DispatchSemaphore(value: 0) - - UNUserNotificationCenter.current().requestAuthorization(options: [.alert, .badge, .sound]) { _, _ in - semaphore.signal() + CompletionWait.completion(timeout: Self.permissionPromptTimeoutSeconds) { complete in + UNUserNotificationCenter.current().requestAuthorization(options: [.alert, .badge, .sound]) { _, _ in + complete() + } } - - _ = semaphore.wait(timeout: .now() + 10) return notificationsStatus() #else return .unavailable @@ -669,11 +669,33 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable #if canImport(CoreLocation) && os(iOS) private final class LocationPermissionDelegate: NSObject, CLLocationManagerDelegate { private let semaphore = DispatchSemaphore(value: 0) + private let lock = NSLock() + private var status: CLAuthorizationStatus? + /// The manager is created inside a `DispatchQueue.main.async` block and would + /// otherwise be released as soon as that block returns — before the callback + /// it exists to receive. `CLLocationManager` does not retain its delegate, so + /// the ownership runs the other way here. + private var manager: CLLocationManager? + + func hold(_ manager: CLLocationManager) { + lock.lock() + self.manager = manager + lock.unlock() + } func locationManagerDidChangeAuthorization(_ manager: CLLocationManager) { + lock.lock() + status = manager.authorizationStatus + lock.unlock() semaphore.signal() } + func observedStatus() -> CLAuthorizationStatus? { + lock.lock() + defer { lock.unlock() } + return status + } + func wait(timeout: TimeInterval) -> DispatchTimeoutResult { semaphore.wait(timeout: .now() + timeout) } diff --git a/Sources/CodeMode/Support/CompletionWait.swift b/Sources/CodeMode/Support/CompletionWait.swift new file mode 100644 index 0000000..38f54f4 --- /dev/null +++ b/Sources/CodeMode/Support/CompletionWait.swift @@ -0,0 +1,55 @@ +import Foundation + +/// Bridges a callback-style system API into the bridges' synchronous world. +/// +/// The pattern this replaces — a `var` written from the framework's completion +/// queue, read after `_ = semaphore.wait(...)` with the timeout result +/// discarded — is a data race on timeout: the late callback writes while the +/// caller reads, and the caller cannot tell "timed out" from "no results". Every +/// wait here either produces the value the callback delivered or throws +/// `BridgeError.timeout`; the box stays alive for a late callback to write into +/// harmlessly. +enum CompletionWait { + /// Runs `operation`, which must call its `complete` argument exactly once, + /// and returns the delivered value. + /// + /// - Throws: `BridgeError.timeout` if `complete` is not called in time. + static func value( + timeout: TimeInterval, + operationName: String, + _ operation: (@escaping @Sendable (Value) -> Void) -> Void + ) throws -> Value { + let box = LockedBox(nil) + let semaphore = DispatchSemaphore(value: 0) + + operation { value in + box.set(value) + semaphore.signal() + } + + guard semaphore.wait(timeout: .now() + timeout) == .success else { + throw BridgeError.timeout(milliseconds: Int(timeout * 1_000)) + } + + guard let value = box.get() else { + throw BridgeError.nativeFailure("\(operationName) completed without a result") + } + + return value + } + + /// Waits for a completion that carries no value, for APIs whose result is + /// read back from the store afterwards. + /// + /// - Returns: true when the callback arrived, false on timeout. Callers that + /// must distinguish the two should use `value(timeout:operationName:_:)`. + @discardableResult + static func completion( + timeout: TimeInterval, + _ operation: (@escaping @Sendable () -> Void) -> Void + ) -> Bool { + let semaphore = DispatchSemaphore(value: 0) + operation { semaphore.signal() } + return semaphore.wait(timeout: .now() + timeout) == .success + } +} diff --git a/Tests/CodeModeTests/CompletionWaitTests.swift b/Tests/CodeModeTests/CompletionWaitTests.swift new file mode 100644 index 0000000..12df116 --- /dev/null +++ b/Tests/CodeModeTests/CompletionWaitTests.swift @@ -0,0 +1,49 @@ +import Foundation +import Testing +@testable import CodeMode + +// The pattern this replaces — a `var` written from a framework's completion +// queue and read after a semaphore wait whose timeout result was discarded — is +// both a data race and an ambiguity: the caller could not tell "timed out" from +// "the API returned nothing". + +@Test func completionWaitReturnsTheDeliveredValue() throws { + let value = try CompletionWait.value(timeout: 5, operationName: "test") { complete in + DispatchQueue.global().async { complete([1, 2, 3]) } + } + #expect(value == [1, 2, 3]) +} + +@Test func completionWaitThrowsRatherThanReturningAnEmptyResult() { + do { + // Never completes: the old shape returned `.array([])`, indistinguishable + // from a genuinely empty fetch. + let value: [Int] = try CompletionWait.value(timeout: 0.1, operationName: "reminders.read") { _ in } + Issue.record("Expected a timeout, got \(value)") + } catch { + #expect(requireBridgeErrorCode(error) == "EXECUTION_TIMEOUT") + } +} + +@Test func completionWaitToleratesALateCallback() throws { + // The late write must land in the box rather than racing a caller's read. + let lateCallbackFinished = DispatchSemaphore(value: 0) + do { + let _: Int = try CompletionWait.value(timeout: 0.05, operationName: "test") { complete in + DispatchQueue.global().asyncAfter(deadline: .now() + 0.2) { + complete(7) + lateCallbackFinished.signal() + } + } + Issue.record("Expected a timeout") + } catch { + #expect(requireBridgeErrorCode(error) == "EXECUTION_TIMEOUT") + } + + #expect(lateCallbackFinished.wait(timeout: .now() + 5) == .success) +} + +@Test func completionWaitReportsWhetherTheCallbackArrived() { + #expect(CompletionWait.completion(timeout: 5) { complete in complete() }) + #expect(CompletionWait.completion(timeout: 0.05) { _ in } == false) +} From 226337a95d15540d2763d38fa1c55b63a6267a15 Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 20:11:07 -0700 Subject: [PATCH 07/18] Work through the P2 correctness and polish findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **One ISO 8601 parser.** Health and Alarm fell back to `.withFractionalSeconds`; EventKit, Notifications, and the system-UI validators used a bare `ISO8601DateFormatter`. So `2026-07-24T10:00:00.000Z` — what JavaScript's `toISOString()` emits, and what models therefore produce by default — parsed in some bridges and not others. Worse, `calendar.read`/`reminders.read` then did `isoDate(...) ?? Date()`, handing the script a plausible but wrong window with no diagnostic. `CodeModeDate` is now the only parser, and a present-but-unparseable date is an error; only an absent one takes the default. **HealthKit stops claiming access it cannot verify.** `requestAuthorization`'s `success` means the sheet flow completed — HealthKit deliberately hides read denial so an app cannot infer a condition from it — but the bridge reported `granted: true`, and mapped `getRequestStatusForAuthorization == .unnecessary` ("nothing left to ask") to `.granted`. Reads then returned [] while the transcript claimed permission. It now reports flow completion, per-type *share* authorization (the only part HealthKit answers), and says plainly that read authorization is undisclosed. The broker's fail-closed `healthKitStatus` is untouched. **HomeKit status no longer prompts.** Constructing `HMHomeManager` is what triggers the TCC dialog, and `homeKitStatus()` built one inside `status(for:)` while `requestHomeKitPermission` called it twice and built a third. Status now answers from a shared manager if one exists and otherwise fails closed with `.notDetermined`, routing the caller to the request path where a prompt belongs. **Network and browser destinations.** Bare single-label hosts (`router`, `intranet`, `wpad`, `nas`) resolve onto the LAN through the DHCP search domain but fell through to "allowed"; they are refused like `*.local`, and an explicit `allowedHosts` entry still wins. `web.ui.present` and `auth.ui.webAuthenticate` now consult `NetworkAccessPolicy` — opening a page in a browser was exempt from the policy governing fetch — and `auth.ui.webAuthenticate` defaults to an ephemeral browser session, so a script-started OAuth flow no longer begins already signed in as the user. **Crash and containment fixes.** `EKEvent.calendar` (`EKCalendar!`, nil for orphaned events) and `EKReminder.title` (`String!`) are nil-coalesced like every neighbouring field. Two force-unwrapped `URL(string:)!` in the public `UIKitSystemUIPresenter` throw instead. A symlink to a *nonexistent* out-of-root target passed containment, because `fileExists` follows links (so the link read as a missing component) and `resolvingSymlinksInPath` leaves a dangling link unchanged; both halves are fixed, with a bounded walk for cycles. **Concurrency honesty.** `Task.isCancelled` was checked from plain GCD workers where it is always false — removed, leaving the controller that `cancel()` actually sets. The watchdog deadline is a `ContinuousClock.Instant`, so an NTP adjustment cannot move it. `JavaScriptExecutionCall` cancels on deinit, so a dropped call stops instead of running to completion in the background, and `result` no longer spawns a `Task.detached` per access. **Diagnostics.** Error-severity diagnostics were recorded but never emitted, so a host watching the stream saw every diagnostic except the ones that mattered most. **Path arguments.** `speech.file.transcribe` and `passkit.pass.add` left the raw script-supplied `path` in the dictionary alongside `resolvedPath`; the raw value is now replaced, so a host adapter reading `path` cannot bypass the path policy. Golden baseline regenerated for the two web-auth metadata strings. --- Sources/CodeMode/API/BridgeModels.swift | 24 ++- Sources/CodeMode/Bridges/AlarmBridge.swift | 8 +- .../Bridges/BigTicketAppleBridges.swift | 7 + .../Bridges/DefaultCapabilityLoader.swift | 2 +- Sources/CodeMode/Bridges/EventKitBridge.swift | 26 ++-- Sources/CodeMode/Bridges/HealthBridge.swift | 69 +++++++-- .../Bridges/NotificationsBridge.swift | 2 +- Sources/CodeMode/Bridges/SystemUIBridge.swift | 16 +- .../Bridges/SystemUICodeModeTools.swift | 4 +- .../Runtime/BridgeInvocationContext.swift | 5 +- Sources/CodeMode/Runtime/BridgeRuntime.swift | 10 +- .../CodeMode/Runtime/ExecutionSupport.swift | 6 +- .../CodeMode/Runtime/ExecutionWatchdog.swift | 17 ++- .../Security/NetworkAccessPolicy.swift | 15 +- Sources/CodeMode/Security/PathPolicy.swift | 43 +++++- .../Security/SystemPermissionBroker.swift | 58 +++++-- Sources/CodeMode/Support/CodeModeDate.swift | 62 ++++++++ .../UIKitSystemUIPresenter+Calendar.swift | 5 +- ...itSystemUIPresenter+CommunicationWeb.swift | 17 ++- Tests/CodeModeTests/PolishFindingsTests.swift | 143 ++++++++++++++++++ .../capability-metadata-golden.json | 4 +- 21 files changed, 463 insertions(+), 80 deletions(-) create mode 100644 Sources/CodeMode/Support/CodeModeDate.swift create mode 100644 Tests/CodeModeTests/PolishFindingsTests.swift diff --git a/Sources/CodeMode/API/BridgeModels.swift b/Sources/CodeMode/API/BridgeModels.swift index 8c2bca5..3e282e3 100644 --- a/Sources/CodeMode/API/BridgeModels.swift +++ b/Sources/CodeMode/API/BridgeModels.swift @@ -355,7 +355,10 @@ public final class JavaScriptExecutionCall: @unchecked Sendable { public var result: JavaScriptExecutionResult { get async throws { let outcome = await withTaskCancellationHandler { - await waitForResultOutcome() + // `resultTask.result` already detaches the await from the calling + // task's cancellation, so the previous `Task.detached` per access + // bought nothing but an extra task. + await resultTask.result } onCancel: { self.cancel() } @@ -365,8 +368,7 @@ public final class JavaScriptExecutionCall: @unchecked Sendable { return result case let .failure(error as CodeModeToolError): throw error - case let .failure(error as CancellationError): - _ = error + case .failure(is CancellationError): throw CodeModeToolError(code: "CANCELLED", message: "Execution cancelled") case let .failure(error): throw error @@ -379,16 +381,12 @@ public final class JavaScriptExecutionCall: @unchecked Sendable { resultTask.cancel() } - private func waitForResultOutcome() async -> Result { - await withCheckedContinuation { continuation in - Task.detached { - do { - continuation.resume(returning: .success(try await self.resultTask.value)) - } catch { - continuation.resume(returning: .failure(error)) - } - } - } + /// A dropped call must not keep running: without this, an execution whose + /// handle nobody holds continues to completion in the background, still + /// touching the filesystem, the network, and system UI on the user's behalf. + deinit { + cancelImpl() + resultTask.cancel() } } diff --git a/Sources/CodeMode/Bridges/AlarmBridge.swift b/Sources/CodeMode/Bridges/AlarmBridge.swift index 4736f96..6931d19 100644 --- a/Sources/CodeMode/Bridges/AlarmBridge.swift +++ b/Sources/CodeMode/Bridges/AlarmBridge.swift @@ -139,13 +139,7 @@ public final class AlarmBridge: @unchecked Sendable { } private func isoDate(_ text: String?) -> Date? { - guard let text else { return nil } - let formatter = ISO8601DateFormatter() - if let date = formatter.date(from: text) { - return date - } - formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - return formatter.date(from: text) + CodeModeDate.parse(text) } #if canImport(AlarmKit) && canImport(SwiftUI) && os(iOS) diff --git a/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift b/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift index 55090e8..1160d8c 100644 --- a/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift +++ b/Sources/CodeMode/Bridges/BigTicketAppleBridges.swift @@ -111,7 +111,12 @@ public final class SpeechBridge: @unchecked Sendable { try ensurePermission(.speechRecognition, context: context) let path = try requireNonEmptyString("path", in: arguments, capability: "speech.file.transcribe") let resolved = try context.pathPolicy.resolve(path: path) + // The raw, script-supplied `path` is replaced rather than accompanied: + // leaving both in the dictionary made every host adapter one + // `arguments.string("path")` away from a path-policy bypass, with nothing + // in the code saying which key was the safe one. var clientArguments = arguments + clientArguments["path"] = .string(resolved.path) clientArguments["resolvedPath"] = .string(resolved.path) try validateOptionalPositiveNumber("timeoutMs", in: arguments, capability: "speech.file.transcribe") return try client.transcribeFile(arguments: clientArguments) @@ -348,7 +353,9 @@ public final class PassKitBridge: @unchecked Sendable { public func addPass(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let path = try requireNonEmptyString("path", in: arguments, capability: "passkit.pass.add") let resolved = try context.pathPolicy.resolve(path: path) + // As above: the raw path does not survive into the client dictionary. var clientArguments = arguments + clientArguments["path"] = .string(resolved.path) clientArguments["resolvedPath"] = .string(resolved.path) return try client.addPass(arguments: clientArguments) } diff --git a/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift b/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift index 187308d..3985f55 100644 --- a/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift +++ b/Sources/CodeMode/Bridges/DefaultCapabilityLoader.swift @@ -94,7 +94,7 @@ struct DefaultCapabilityRegistrationBuilder { self.health = HealthBridge() self.home = HomeBridge() self.media = MediaBridge() - self.systemUI = SystemUIBridge() + self.systemUI = SystemUIBridge(networkAccessPolicy: networkAccessPolicy) self.eventInbox = eventInbox self.cloudKit = CloudKitBridge(client: cloudKitClient, eventInbox: eventInbox) self.remoteNotifications = RemoteNotificationsBridge(client: remoteNotificationsClient, eventInbox: eventInbox) diff --git a/Sources/CodeMode/Bridges/EventKitBridge.swift b/Sources/CodeMode/Bridges/EventKitBridge.swift index cd7cdb8..b0b6a98 100644 --- a/Sources/CodeMode/Bridges/EventKitBridge.swift +++ b/Sources/CodeMode/Bridges/EventKitBridge.swift @@ -17,8 +17,13 @@ public final class EventKitBridge: @unchecked Sendable { #if canImport(EventKit) let store = EKEventStore() - let start = isoDate(arguments.string("start")) ?? Date() - let end = isoDate(arguments.string("end")) ?? Calendar.current.date(byAdding: .day, value: 14, to: start) ?? start + // A value that is present but unparseable is an error, not a reason to + // fall back to now/+14d: that handed the script a plausible but wrong + // window with no diagnostic to notice it by. + let start = try CodeModeDate.optional(arguments.string("start"), argument: "start", capability: "calendar.read") ?? Date() + let end = try CodeModeDate.optional(arguments.string("end"), argument: "end", capability: "calendar.read") + ?? Calendar.current.date(byAdding: .day, value: 14, to: start) + ?? start let limit = arguments.int("limit") ?? 50 let calendars = try resolveCalendars( from: arguments, @@ -100,8 +105,8 @@ public final class EventKitBridge: @unchecked Sendable { #if canImport(EventKit) let store = EKEventStore() let includeCompleted = arguments.bool("includeCompleted") ?? false - let start = isoDate(arguments.string("start")) - let end = isoDate(arguments.string("end")) + let start = try CodeModeDate.optional(arguments.string("start"), argument: "start", capability: "reminders.read") + let end = try CodeModeDate.optional(arguments.string("end"), argument: "end", capability: "reminders.read") let limit = max(1, arguments.int("limit") ?? 50) let calendars = try resolveCalendars( from: arguments, @@ -198,8 +203,7 @@ public final class EventKitBridge: @unchecked Sendable { } private func isoDate(_ text: String?) -> Date? { - guard let text, text.isEmpty == false else { return nil } - return ISO8601DateFormatter().date(from: text) + CodeModeDate.parse(text) } private func eventOperation(_ arguments: [String: JSONValue]) throws -> CalendarWriteOperation { @@ -494,8 +498,11 @@ public final class EventKitBridge: @unchecked Sendable { "startDate": .string(event.startDate.ISO8601Format()), "endDate": .string(event.endDate.ISO8601Format()), "notes": .string(event.notes ?? ""), - "calendarIdentifier": .string(event.calendar.calendarIdentifier), - "calendarTitle": .string(event.calendar.title), + // `EKEvent.calendar` is `EKCalendar!` and is nil for an orphaned + // event — every neighboring field here is nil-coalesced, and this one + // crashed the execution thread. + "calendarIdentifier": .string(event.calendar?.calendarIdentifier ?? ""), + "calendarTitle": .string(event.calendar?.title ?? ""), "location": .string(event.location ?? ""), "url": .string(event.url?.absoluteString ?? ""), "isAllDay": .bool(event.isAllDay), @@ -505,7 +512,8 @@ public final class EventKitBridge: @unchecked Sendable { private static func reminderJSON(_ reminder: EKReminder) -> JSONValue { var object: [String: JSONValue] = [ "identifier": .string(reminder.calendarItemIdentifier), - "title": .string(reminder.title), + // `EKReminder.title` is `String!`. + "title": .string(reminder.title ?? ""), "isCompleted": .bool(reminder.isCompleted), "dueDate": .string(reminder.dueDateComponents?.date?.ISO8601Format() ?? ""), "completionDate": .string(reminder.completionDate?.ISO8601Format() ?? ""), diff --git a/Sources/CodeMode/Bridges/HealthBridge.swift b/Sources/CodeMode/Bridges/HealthBridge.swift index 0732e72..7f584ba 100644 --- a/Sources/CodeMode/Bridges/HealthBridge.swift +++ b/Sources/CodeMode/Bridges/HealthBridge.swift @@ -26,11 +26,21 @@ public final class HealthBridge: @unchecked Sendable { let writeTypes = try requestedWriteTypes(from: writeNames) try ensureAuthorization(toShare: writeTypes, read: readTypes, context: context, forceRequest: true) + // `requestAuthorization`'s `success` means the sheet flow completed, not + // that anything was granted — HealthKit deliberately refuses to disclose + // read authorization so an app cannot infer a health condition from a + // denial. Reporting `granted: true` here was a fabrication that left the + // transcript claiming access while reads returned []. Share types are the + // only ones with an answerable status, so those are the only ones + // reported. return .object([ - "status": .string(PermissionStatus.granted.rawValue), - "granted": .bool(true), + "status": .string("completed"), + "authorizationFlowCompleted": .bool(true), "readTypes": .array(readNames.map(JSONValue.string)), "writeTypes": .array(writeNames.map(JSONValue.string)), + "writeAuthorization": .object(writeAuthorizationStatuses(for: writeNames)), + "readAuthorizationDisclosed": .bool(false), + "note": .string("HealthKit does not disclose read authorization. An empty health.read result may mean the user denied that type rather than that no samples exist."), ]) #else _ = arguments @@ -150,15 +160,7 @@ public final class HealthBridge: @unchecked Sendable { } private func parseISODate(_ value: String?) -> Date? { - guard let value else { return nil } - - let formatter = ISO8601DateFormatter() - if let date = formatter.date(from: value) { - return date - } - - formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - return formatter.date(from: value) + CodeModeDate.parse(value) } #if canImport(HealthKit) @@ -320,6 +322,44 @@ public final class HealthBridge: @unchecked Sendable { return types } + /// Per-type share authorization, which is the only part HealthKit answers + /// honestly. Read authorization is deliberately undisclosed by the framework. + private func writeAuthorizationStatuses(for names: [String]) -> [String: JSONValue] { + var statuses: [String: JSONValue] = [:] + for name in names { + guard let spec = try? writeSpec(for: name), + let quantityType = HKObjectType.quantityType(forIdentifier: spec.identifier) + else { + continue + } + statuses[name] = .string(Self.shareStatusName(healthStore.authorizationStatus(for: quantityType))) + } + return statuses + } + + /// `.granted` only when every requested share type is actually authorized; + /// otherwise `.notDetermined`, because HealthKit will not say more. + private func resolvedShareAuthorization(for toShare: Set) -> PermissionStatus { + guard toShare.isEmpty == false else { + return .notDetermined + } + let allShared = toShare.allSatisfy { healthStore.authorizationStatus(for: $0) == .sharingAuthorized } + return allShared ? .granted : .notDetermined + } + + private static func shareStatusName(_ status: HKAuthorizationStatus) -> String { + switch status { + case .sharingAuthorized: + return PermissionStatus.granted.rawValue + case .sharingDenied: + return PermissionStatus.denied.rawValue + case .notDetermined: + return PermissionStatus.notDetermined.rawValue + @unknown default: + return PermissionStatus.unavailable.rawValue + } + } + private func ensureAuthorization( toShare: Set, read: Set, @@ -340,7 +380,12 @@ public final class HealthBridge: @unchecked Sendable { let requestStatus = try authorizationRequestStatus(toShare: toShare, read: read) switch requestStatus { case .unnecessary: - context.recordPermission(.healthKit, status: .granted) + // `.unnecessary` means "there is nothing left to ask the user", not + // "access was granted" — for read types the user may well have + // denied. Record what is actually known: granted only when every + // share type says so, otherwise leave it undetermined rather than + // writing a granted claim into the transcript. + context.recordPermission(.healthKit, status: resolvedShareAuthorization(for: toShare)) case .shouldRequest, .unknown: context.recordPermission(.healthKit, status: .notDetermined) let granted = try requestAuthorization(toShare: toShare, read: read) diff --git a/Sources/CodeMode/Bridges/NotificationsBridge.swift b/Sources/CodeMode/Bridges/NotificationsBridge.swift index da6fc2a..178f605 100644 --- a/Sources/CodeMode/Bridges/NotificationsBridge.swift +++ b/Sources/CodeMode/Bridges/NotificationsBridge.swift @@ -228,7 +228,7 @@ public final class NotificationsBridge: @unchecked Sendable { private func isoDate(_ text: String?) -> Date? { guard let text else { return nil } - return ISO8601DateFormatter().date(from: text) + return CodeModeDate.parse(text) } #if canImport(UserNotifications) diff --git a/Sources/CodeMode/Bridges/SystemUIBridge.swift b/Sources/CodeMode/Bridges/SystemUIBridge.swift index a5b59bb..1d63159 100644 --- a/Sources/CodeMode/Bridges/SystemUIBridge.swift +++ b/Sources/CodeMode/Bridges/SystemUIBridge.swift @@ -1,7 +1,11 @@ import Foundation public final class SystemUIBridge: @unchecked Sendable { - public init() {} + private let networkAccessPolicy: NetworkAccessPolicy + + public init(networkAccessPolicy: NetworkAccessPolicy = .standard) { + self.networkAccessPolicy = networkAccessPolicy + } public func pickCalendar(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { try validateCalendarPickerArguments(arguments) @@ -188,12 +192,11 @@ public final class SystemUIBridge: @unchecked Sendable { } private func validateCalendarEventArguments(_ arguments: [String: JSONValue]) throws { - let formatter = ISO8601DateFormatter() for key in ["start", "end"] { guard let value = arguments.string(key) else { continue } - guard formatter.date(from: value) != nil else { + guard CodeModeDate.parse(value) != nil else { throw BridgeError.invalidArguments("calendar.ui.presentNewEvent requires \(key) to be an ISO8601 timestamp when provided") } } @@ -447,6 +450,13 @@ public final class SystemUIBridge: @unchecked Sendable { else { throw BridgeError.invalidArguments("\(capability) \(key) must be an absolute HTTP(S) URL") } + + // Destinations opened in a browser were exempt from the policy that + // governs fetch, so a script blocked from reaching an origin could still + // put it in front of the user — or start an OAuth flow against it. + if let reason = networkAccessPolicy.violationReason(for: url) { + throw BridgeError.networkPolicyViolation("\(capability) \(key): \(reason)") + } } private func validateNonemptyString(_ arguments: [String: JSONValue], key: String, capability: String) throws { diff --git a/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift b/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift index b4d47d7..398af2d 100644 --- a/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift +++ b/Sources/CodeMode/Bridges/SystemUICodeModeTools.swift @@ -327,7 +327,7 @@ struct WebUIPresentTool: BuiltInCodeModeTool { @BuiltInCodeMode(.authUIWebAuthenticate, path: "apple.auth.webAuthenticate") struct AuthUIWebAuthenticateTool: BuiltInCodeModeTool { static let codeModeTitle = "Authenticate with system web UI" - static let codeModeSummary = "Start an ASWebAuthenticationSession for OAuth-style browser authentication." + static let codeModeSummary = "Start an ASWebAuthenticationSession for OAuth-style browser authentication. Runs in a private browser session by default and the URL must satisfy the host's network access policy." static let codeModeTags = ["auth", "oauth", "web", "browser", "system-ui"] static let codeModeExample = "await apple.auth.webAuthenticate({ url: 'https://example.com/oauth', callbackURLScheme: 'myapp' })" static let codeModeResultSummary = "Object with action callback/cancelled and callbackURL when available." @@ -337,7 +337,7 @@ struct AuthUIWebAuthenticateTool: BuiltInCodeModeTool { var url: String @ToolParam("Optional custom URL scheme that completes the session.") var callbackURLScheme: String? - @ToolParam("Whether to prefer a private browser session; default false.") + @ToolParam("Whether to use a private browser session; default true. Set false only when the flow must reuse the user's existing browser session.") var prefersEphemeralSession: Bool? @ToolParam("Optional timeout for waiting on callback/cancellation.") var timeoutMs: Double? diff --git a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift index 170defb..b805d09 100644 --- a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift +++ b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift @@ -119,7 +119,10 @@ public final class BridgeInvocationContext: @unchecked Sendable { } func checkCancellation() throws { - if cancellationController.isCancelled || Task.isCancelled { + // Bridge handlers run on a GCD worker, not inside a Task, so + // `Task.isCancelled` was always false here. The controller is the signal + // `JavaScriptExecutionCall.cancel()` actually sets. + if cancellationController.isCancelled { throw BridgeError.cancelled } } diff --git a/Sources/CodeMode/Runtime/BridgeRuntime.swift b/Sources/CodeMode/Runtime/BridgeRuntime.swift index b7a1106..95f7873 100644 --- a/Sources/CodeMode/Runtime/BridgeRuntime.swift +++ b/Sources/CodeMode/Runtime/BridgeRuntime.swift @@ -557,8 +557,12 @@ final class BridgeRuntime: @unchecked Sendable { cancelMessage: String ) throws -> SettlementState { while true { - if cancellationController.isCancelled || Task.isCancelled { - cancellationController.cancel() + // `Task.isCancelled` is not checked here: this runs on a plain GCD + // worker with no surrounding Task, so it was always false — dead code + // that read as a second layer of cancellation support. The + // controller, which `JavaScriptExecutionCall.cancel()` sets, is the + // real signal. + if cancellationController.isCancelled { throw toolError(code: "CANCELLED", message: cancelMessage, transcript: invocationContext) } @@ -577,7 +581,7 @@ final class BridgeRuntime: @unchecked Sendable { throw toolError(code: timeoutCode, message: timeoutMessage, transcript: invocationContext) } } - if Date() >= watchdog.deadline { + if watchdog.hasPassedDeadline { throw toolError(code: timeoutCode, message: timeoutMessage, transcript: invocationContext) } Thread.sleep(forTimeInterval: 0.01) diff --git a/Sources/CodeMode/Runtime/ExecutionSupport.swift b/Sources/CodeMode/Runtime/ExecutionSupport.swift index 6e49650..8491f97 100644 --- a/Sources/CodeMode/Runtime/ExecutionSupport.swift +++ b/Sources/CodeMode/Runtime/ExecutionSupport.swift @@ -78,7 +78,11 @@ final class ExecutionTranscript: @unchecked Sendable { } lock.unlock() - if accepted, diagnostic.severity != .error { + // Error-severity diagnostics used to be recorded but never emitted, so a + // host watching the stream saw every diagnostic *except* the ones that + // mattered most. They are on the result either way; the stream is the + // live view and should not be the incomplete one. + if accepted { emitEvent(.diagnostic(diagnostic)) } } diff --git a/Sources/CodeMode/Runtime/ExecutionWatchdog.swift b/Sources/CodeMode/Runtime/ExecutionWatchdog.swift index a9b8ea4..dba24e7 100644 --- a/Sources/CodeMode/Runtime/ExecutionWatchdog.swift +++ b/Sources/CodeMode/Runtime/ExecutionWatchdog.swift @@ -40,21 +40,28 @@ final class ExecutionWatchdog: @unchecked Sendable { fileprivate static let checkInterval: TimeInterval = 0.05 private let lock = NSLock() - private var deadlineValue: Date + /// `ContinuousClock`, not `Date`: a wall-clock deadline moves when NTP + /// adjusts the clock, which can push a timeout arbitrarily far out (or fire it + /// immediately) while a script is running. + private var deadlineValue: ContinuousClock.Instant private var terminationValue: Termination? private let cancellationController: ExecutionCancellationController init(timeoutMs: Int, cancellationController: ExecutionCancellationController) { - self.deadlineValue = Date().addingTimeInterval(Double(timeoutMs) / 1_000) + self.deadlineValue = ContinuousClock.now.advanced(by: .milliseconds(timeoutMs)) self.cancellationController = cancellationController } - var deadline: Date { + var deadline: ContinuousClock.Instant { lock.lock() defer { lock.unlock() } return deadlineValue } + var hasPassedDeadline: Bool { + ContinuousClock.now >= deadline + } + var termination: Termination? { lock.lock() defer { lock.unlock() } @@ -67,7 +74,7 @@ final class ExecutionWatchdog: @unchecked Sendable { /// user-defined getters/`toJSON`), while a runaway getter is still terminated. func rearm(timeoutMs: Int) { lock.lock() - deadlineValue = Date().addingTimeInterval(Double(timeoutMs) / 1_000) + deadlineValue = ContinuousClock.now.advanced(by: .milliseconds(timeoutMs)) lock.unlock() } @@ -100,7 +107,7 @@ final class ExecutionWatchdog: @unchecked Sendable { terminationValue = .cancelled return true } - if Date() >= deadlineValue { + if ContinuousClock.now >= deadlineValue { terminationValue = .timedOut return true } diff --git a/Sources/CodeMode/Security/NetworkAccessPolicy.swift b/Sources/CodeMode/Security/NetworkAccessPolicy.swift index f391301..6bb8e80 100644 --- a/Sources/CodeMode/Security/NetworkAccessPolicy.swift +++ b/Sources/CodeMode/Security/NetworkAccessPolicy.swift @@ -5,8 +5,11 @@ import Foundation /// /// The default (`.standard`) blocks loopback, private-range, link-local, and /// other non-public destinations so agent-authored JavaScript cannot reach -/// `127.0.0.1`, cloud metadata endpoints such as `169.254.169.254`, or hosts on -/// the local network, and caps buffered response bodies at 10 MB. +/// `127.0.0.1`, cloud metadata endpoints such as `169.254.169.254`, or names that +/// resolve onto the local network, and caps buffered response bodies at 10 MB. +/// "Names that resolve onto the local network" covers `localhost`, +/// `*.local`/`*.localhost`/`*.internal`, and bare single-label hosts such as +/// `router` or `intranet`, which reach the LAN through the DHCP search domain. /// /// Host matching is by URL host only; DNS resolution is not performed, so a /// public hostname that resolves to a private address is not detected. Hosts @@ -155,6 +158,14 @@ public struct NetworkAccessPolicy: Sendable, Equatable { return isPrivateOrLocal(ipv6: ipv6) } + // A bare single-label name — `router`, `intranet`, `nas`, `wpad` — is not + // a public destination. It resolves through the DHCP-supplied search + // domain, so it lands on the local network exactly like `*.local` does, + // and it previously fell straight through to "allowed". + if host.contains(".") == false { + return true + } + return false } diff --git a/Sources/CodeMode/Security/PathPolicy.swift b/Sources/CodeMode/Security/PathPolicy.swift index 08af30b..45746b0 100644 --- a/Sources/CodeMode/Security/PathPolicy.swift +++ b/Sources/CodeMode/Security/PathPolicy.swift @@ -122,15 +122,54 @@ public struct DefaultPathPolicy: PathPolicy { var missingComponents: [String] = [] let fileManager = FileManager.default - while fileManager.fileExists(atPath: existingAncestor.path) == false, + // `fileExists` follows symlinks, so a symlink pointing at a *nonexistent* + // out-of-root target reported "missing" and was treated as a component to + // re-append rather than a link to resolve — admitting it. Checking for the + // link itself keeps a dangling symlink in the resolution path, where + // `resolvingSymlinksInPath` sends containment to the real target. + while Self.isMissingComponent(existingAncestor, fileManager: fileManager), existingAncestor.path != existingAncestor.deletingLastPathComponent().path { missingComponents.insert(existingAncestor.lastPathComponent, at: 0) existingAncestor.deleteLastPathComponent() } - let resolvedAncestor = existingAncestor.resolvingSymlinksInPath().standardizedFileURL + let resolvedAncestor = Self.followingDanglingSymlink( + existingAncestor.resolvingSymlinksInPath().standardizedFileURL, + fileManager: fileManager + ) return missingComponents.reduce(resolvedAncestor) { partial, component in partial.appendingPathComponent(component) }.standardizedFileURL } + + /// True when nothing exists at this path — not even a broken symlink. + /// `FileManager.fileExists` follows links and so answers false for one. + private static func isMissingComponent(_ url: URL, fileManager: FileManager) -> Bool { + if fileManager.fileExists(atPath: url.path) { + return false + } + // `attributesOfItem` does not traverse the final symlink, so a dangling + // link still has attributes here. + return (try? fileManager.attributesOfItem(atPath: url.path)) == nil + } + + /// Resolves a symlink whose target does not exist. + /// + /// `URL.resolvingSymlinksInPath()` only rewrites links it can follow, so a + /// link to a nonexistent out-of-root target came back unchanged and passed + /// containment. Containment must be judged on where the link *points*. + private static func followingDanglingSymlink(_ url: URL, fileManager: FileManager) -> URL { + var current = url + // Bounded: a symlink cycle would otherwise spin here. + for _ in 0..<16 { + guard let destination = try? fileManager.destinationOfSymbolicLink(atPath: current.path) else { + return current + } + let next = destination.hasPrefix("/") + ? URL(fileURLWithPath: destination) + : current.deletingLastPathComponent().appendingPathComponent(destination) + current = next.standardizedFileURL.resolvingSymlinksInPath().standardizedFileURL + } + return current + } } diff --git a/Sources/CodeMode/Security/SystemPermissionBroker.swift b/Sources/CodeMode/Security/SystemPermissionBroker.swift index b2cad2a..cd3b254 100644 --- a/Sources/CodeMode/Security/SystemPermissionBroker.swift +++ b/Sources/CodeMode/Security/SystemPermissionBroker.swift @@ -331,9 +331,47 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable #endif } + /// Reads HomeKit authorization *without* constructing a manager. + /// + /// Constructing `HMHomeManager` is itself what triggers the HomeKit TCC + /// prompt, so the previous implementation put a user-facing dialog inside a + /// status check — and `requestHomeKitPermission` called it twice, then built a + /// third manager. A status query now answers from the shared manager if one + /// already exists and otherwise fails closed with `.notDetermined`, which + /// routes the caller through the request path where a prompt belongs. private func homeKitStatus() -> PermissionStatus { #if canImport(HomeKit) - let status = HMHomeManager().authorizationStatus + guard let manager = Self.existingHomeManager() else { + return .notDetermined + } + return Self.mapHomeKitAuthorization(manager.authorizationStatus) + #else + return .unavailable + #endif + } + + #if canImport(HomeKit) + /// Process-wide so repeated permission work reuses one manager instead of + /// re-triggering the prompt. + private static let homeManagerBox = LockedBox(nil) + + private static func existingHomeManager() -> HMHomeManager? { + homeManagerBox.get() + } + + /// Creates the shared manager if needed. This is the call that prompts. + private static func makeHomeManager(delegate: HMHomeManagerDelegate) -> HMHomeManager { + if let existing = homeManagerBox.get() { + existing.delegate = delegate + return existing + } + let manager = HMHomeManager() + manager.delegate = delegate + homeManagerBox.set(manager) + return manager + } + + private static func mapHomeKitAuthorization(_ status: HMHomeManagerAuthorizationStatus) -> PermissionStatus { if status.contains(.authorized) { return .granted } @@ -344,10 +382,8 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable return .notDetermined } return .unavailable - #else - return .unavailable - #endif } + #endif private func speechRecognitionStatus() -> PermissionStatus { #if canImport(Speech) @@ -560,15 +596,17 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestHomeKitPermission() -> PermissionStatus { #if canImport(HomeKit) - if homeKitStatus() != .notDetermined { - return homeKitStatus() + // One status read, not two, and one manager, not three: each construction + // is a potential prompt. + let current = homeKitStatus() + if current != .notDetermined, Self.existingHomeManager() != nil { + return current } let delegate = HomeKitPermissionDelegate() - let manager = HMHomeManager() - manager.delegate = delegate - _ = delegate.wait(timeout: 10) - return homeKitStatus() + let manager = Self.makeHomeManager(delegate: delegate) + _ = delegate.wait(timeout: Self.permissionPromptTimeoutSeconds) + return Self.mapHomeKitAuthorization(manager.authorizationStatus) #else return .unavailable #endif diff --git a/Sources/CodeMode/Support/CodeModeDate.swift b/Sources/CodeMode/Support/CodeModeDate.swift new file mode 100644 index 0000000..a7f3fc8 --- /dev/null +++ b/Sources/CodeMode/Support/CodeModeDate.swift @@ -0,0 +1,62 @@ +import Foundation + +/// The single ISO 8601 parser for every bridge. +/// +/// Two spellings were previously in circulation: Health and Alarm tried +/// `.withFractionalSeconds` as a fallback, while EventKit, Notifications, and the +/// system-UI validators used a bare `ISO8601DateFormatter`. So +/// `2026-07-24T10:00:00.000Z` — a spelling models emit constantly, and what +/// `Date.toISOString()` produces in JavaScript — parsed in some bridges and not +/// others. +enum CodeModeDate { + /// Parses an ISO 8601 timestamp with or without fractional seconds. + static func parse(_ text: String?) -> Date? { + guard let text else { + return nil + } + + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard trimmed.isEmpty == false else { + return nil + } + + let formatter = ISO8601DateFormatter() + if let date = formatter.date(from: trimmed) { + return date + } + + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return formatter.date(from: trimmed) + } + + /// Parses a timestamp that the caller requires, throwing a repairable error + /// rather than substituting a default. + /// + /// Silently falling back to "now" (or "now + 14 days") gave the script a + /// plausible but wrong window with no diagnostic — the worst failure mode + /// available, because the model has no way to notice it happened. + static func require(_ text: String?, argument: String, capability: String) throws -> Date { + guard let text, text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false else { + throw BridgeError.invalidArguments("\(capability) requires '\(argument)'") + } + guard let date = parse(text) else { + throw BridgeError.invalidArguments( + "\(capability) could not parse '\(argument)' as an ISO 8601 timestamp: \"\(text)\". Use a form like 2026-07-24T10:00:00Z." + ) + } + return date + } + + /// Parses an optional timestamp, throwing when a value is present but + /// unparseable. A missing value returns nil and the caller's default applies. + static func optional(_ text: String?, argument: String, capability: String) throws -> Date? { + guard let text, text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == false else { + return nil + } + return try require(text, argument: argument, capability: capability) + } + + static func string(from date: Date) -> String { + ISO8601DateFormatter().string(from: date) + } +} diff --git a/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift b/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift index c3e95b3..f4d8633 100644 --- a/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift +++ b/Sources/CodeMode/Support/UIKitSystemUIPresenter+Calendar.swift @@ -96,7 +96,6 @@ extension UIKitSystemUIPresenter { public func presentNewCalendarEvent(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let timeoutMs = arguments.int("timeoutMs") ?? Self.defaultTimeoutMs - let formatter = ISO8601DateFormatter() let token = UUID() return try runUIOperation(timeoutMs: timeoutMs, onTimeout: { self.releaseCoordinator(token) }) { complete in @@ -108,9 +107,9 @@ extension UIKitSystemUIPresenter { event.notes = arguments.string("notes") event.location = arguments.string("location") - let startDate = arguments.string("start").flatMap { formatter.date(from: $0) } ?? Date() + let startDate = CodeModeDate.parse(arguments.string("start")) ?? Date() event.startDate = startDate - event.endDate = arguments.string("end").flatMap { formatter.date(from: $0) } + event.endDate = CodeModeDate.parse(arguments.string("end")) ?? Calendar.current.date(byAdding: .hour, value: 1, to: startDate) ?? startDate event.calendar = store.defaultCalendarForNewEvents diff --git a/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift b/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift index 8fd4a7c..e3a5f8b 100644 --- a/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift +++ b/Sources/CodeMode/Support/UIKitSystemUIPresenter+CommunicationWeb.swift @@ -365,7 +365,12 @@ extension UIKitSystemUIPresenter { public func presentWeb(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let timeoutMs = arguments.int("timeoutMs") ?? Self.defaultTimeoutMs - let url = URL(string: arguments.string("url") ?? "")! + // The bridge parses the URL first, but UIKitSystemUIPresenter is public + // API and a host may call it directly — a force unwrap here crashes the + // app on a malformed string. + guard let url = URL(string: arguments.string("url") ?? "") else { + throw BridgeError.invalidArguments("web.ui.present requires an absolute 'url'") + } let token = UUID() #if os(iOS) @@ -402,7 +407,9 @@ extension UIKitSystemUIPresenter { public func authenticateWeb(arguments: [String: JSONValue], context: BridgeInvocationContext) throws -> JSONValue { let timeoutMs = arguments.int("timeoutMs") ?? Self.defaultTimeoutMs - let url = URL(string: arguments.string("url") ?? "")! + guard let url = URL(string: arguments.string("url") ?? "") else { + throw BridgeError.invalidArguments("auth.ui.webAuthenticate requires an absolute 'url'") + } let callbackURLScheme = arguments.string("callbackURLScheme") let token = UUID() @@ -446,7 +453,11 @@ extension UIKitSystemUIPresenter { } } session.presentationContextProvider = coordinator - session.prefersEphemeralWebBrowserSession = arguments.bool("prefersEphemeralSession") ?? false + // Ephemeral by default: a script-initiated OAuth flow inside the + // user's live Safari session starts out already signed in as the + // user and hands the callback URL — with its code or token — to + // the script. Opting out is a deliberate host/script choice. + session.prefersEphemeralWebBrowserSession = arguments.bool("prefersEphemeralSession") ?? true coordinator.session = session self.retainCoordinator(coordinator, token: token) diff --git a/Tests/CodeModeTests/PolishFindingsTests.swift b/Tests/CodeModeTests/PolishFindingsTests.swift new file mode 100644 index 0000000..ac3f7cb --- /dev/null +++ b/Tests/CodeModeTests/PolishFindingsTests.swift @@ -0,0 +1,143 @@ +import Foundation +import Testing +@testable import CodeMode + +// MARK: - One ISO 8601 parser + +@Test func iso8601ParsingAcceptsFractionalSeconds() { + // `Date.toISOString()` in JavaScript always emits milliseconds, so this is + // the spelling a model produces by default. A bare ISO8601DateFormatter + // rejects it, and the bridges that used one silently substituted "now". + #expect(CodeModeDate.parse("2026-07-24T10:00:00.000Z") != nil) + #expect(CodeModeDate.parse("2026-07-24T10:00:00Z") != nil) + #expect(CodeModeDate.parse("2026-07-24T10:00:00+01:00") != nil) + #expect(CodeModeDate.parse(" 2026-07-24T10:00:00Z ") != nil) + #expect(CodeModeDate.parse("not a date") == nil) + #expect(CodeModeDate.parse("") == nil) + #expect(CodeModeDate.parse(nil) == nil) +} + +@Test func requiredDatesFailInsteadOfSubstitutingADefault() { + #expect(throws: (any Error).self) { + _ = try CodeModeDate.require("yesterday", argument: "start", capability: "calendar.read") + } + #expect(throws: (any Error).self) { + _ = try CodeModeDate.require(nil, argument: "start", capability: "calendar.read") + } + // A present-but-unparseable optional is still an error; only absence defaults. + #expect(throws: (any Error).self) { + _ = try CodeModeDate.optional("soon", argument: "end", capability: "calendar.read") + } +} + +@Test func absentOptionalDatesStillFallBackToTheCallersDefault() throws { + let absent = try CodeModeDate.optional(nil, argument: "end", capability: "calendar.read") + #expect(absent == nil) +} + +// MARK: - Network policy + +@Test func standardPolicyBlocksBareSingleLabelHosts() { + let policy = NetworkAccessPolicy.standard + // These resolve onto the LAN through the DHCP search domain, exactly like + // *.local, but previously fell through to "allowed". + for host in ["router", "intranet", "wpad", "nas", "printer"] { + let url = URL(string: "http://\(host)/admin")! + #expect(policy.violationReason(for: url) != nil, "expected \(host) to be refused") + } + + // Ordinary public hosts are unaffected. + #expect(policy.violationReason(for: URL(string: "https://example.com/x")!) == nil) + #expect(policy.violationReason(for: URL(string: "https://api.example.co.uk/x")!) == nil) +} + +@Test func hostsCanStillAllowlistASingleLabelHost() { + let policy = NetworkAccessPolicy(allowedHosts: ["router"]) + #expect(policy.violationReason(for: URL(string: "http://router/admin")!) == nil) +} + +// MARK: - System UI destinations + +@Test func webPresentationRespectsTheNetworkAccessPolicy() throws { + let bridge = SystemUIBridge(networkAccessPolicy: .standard) + let (context, sandbox) = try makeInvocationContext() + defer { cleanup(sandbox) } + + // Opening a page in a browser was exempt from the policy governing fetch, so + // a blocked origin could still be put in front of the user — or be the target + // of a script-started OAuth flow. + for capability in ["web", "auth"] { + let arguments: [String: JSONValue] = ["url": .string("http://169.254.169.254/latest/meta-data")] + do { + if capability == "web" { + _ = try bridge.presentWeb(arguments: arguments, context: context) + } else { + _ = try bridge.authenticateWeb(arguments: arguments, context: context) + } + Issue.record("Expected \(capability) presentation to be refused by the network policy") + } catch { + #expect(requireBridgeErrorCode(error) == "NETWORK_POLICY_VIOLATION") + } + } +} + +// MARK: - Path containment + +@Test func danglingSymlinksDoNotPassContainment() throws { + let sandbox = try makeTestSandbox() + defer { cleanup(sandbox) } + + let outsideRoot = sandbox.root.appendingPathComponent("outside", isDirectory: true) + let policy = DefaultPathPolicy( + config: PathPolicyConfig(tmpRoot: sandbox.tmp, cachesRoot: sandbox.caches, documentsRoot: sandbox.documents) + ) + + // The target deliberately does not exist: `fileExists` follows symlinks and + // so reported the link as a *missing component*, which was re-appended to the + // resolved ancestor and admitted instead of being resolved out of the root. + let link = sandbox.tmp.appendingPathComponent("escape") + try FileManager.default.createSymbolicLink( + at: link, + withDestinationURL: outsideRoot.appendingPathComponent("secret.txt") + ) + + #expect(throws: (any Error).self) { + _ = try policy.resolve(path: "tmp:escape") + } +} + +@Test func allowedRootsAreReportedByTheDefaultPolicy() throws { + let sandbox = try makeTestSandbox() + defer { cleanup(sandbox) } + + let policy = DefaultPathPolicy( + config: PathPolicyConfig(tmpRoot: sandbox.tmp, cachesRoot: sandbox.caches, documentsRoot: sandbox.documents) + ) + let documentsRoot = try policy.resolve(path: "documents:") + let tmpRoot = try policy.resolve(path: "tmp:") + let insideTmp = try policy.resolve(path: "tmp:file.txt") + + #expect(policy.allowedRoots.count == 3) + #expect(policy.isAllowedRoot(documentsRoot)) + #expect(policy.isAllowedRoot(tmpRoot)) + #expect(policy.isAllowedRoot(insideTmp) == false) +} + +// MARK: - Diagnostics reach the stream + +@Test func errorSeverityDiagnosticsAreEmittedToTheEventStream() { + let received = LockedBox<[ToolDiagnostic]>([]) + let transcript = ExecutionTranscript { event in + if case let .diagnostic(diagnostic) = event { + received.set(received.get() + [diagnostic]) + } + } + + transcript.record(diagnostic: ToolDiagnostic(severity: .info, code: "I", message: "info")) + transcript.record(diagnostic: ToolDiagnostic(severity: .warning, code: "W", message: "warning")) + // Previously filtered out, so a host watching the stream saw every diagnostic + // except the ones that mattered most. + transcript.record(diagnostic: ToolDiagnostic(severity: .error, code: "E", message: "error")) + + #expect(received.get().map(\.code) == ["I", "W", "E"]) +} diff --git a/Tests/CodeModeTests/capability-metadata-golden.json b/Tests/CodeModeTests/capability-metadata-golden.json index 8569b96..8e94ec8 100644 --- a/Tests/CodeModeTests/capability-metadata-golden.json +++ b/Tests/CodeModeTests/capability-metadata-golden.json @@ -532,7 +532,7 @@ }, "argumentHints" : { "callbackURLScheme" : "Optional custom URL scheme that completes the session.", - "prefersEphemeralSession" : "Whether to prefer a private browser session; default false.", + "prefersEphemeralSession" : "Whether to use a private browser session; default true. Set false only when the flow must reuse the user's existing browser session.", "timeoutMs" : "Optional timeout for waiting on callback\/cancellation.", "url" : "Absolute HTTP(S) authentication URL." }, @@ -559,7 +559,7 @@ ], "resultSummary" : "Object with action callback\/cancelled and callbackURL when available.", - "summary" : "Start an ASWebAuthenticationSession for OAuth-style browser authentication.", + "summary" : "Start an ASWebAuthenticationSession for OAuth-style browser authentication. Runs in a private browser session by default and the URL must satisfy the host's network access policy.", "tags" : [ "auth", "oauth", From 8b84ae36cdf5ec22b3c028248f69a6533c2a4bad Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 20:13:52 -0700 Subject: [PATCH 08/18] Stop multi-step eval scenarios from masking a failed step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `EvalRunner` overwrote `executionOutput`/`Logs`/`Diagnostics`/`observedError` on every step and did not stop on an intermediate error, so validation only ever saw the last one: a scenario could pass while step 1 failed in a way step 2 masked, and step 1's logs — the evidence for why — were discarded before anyone could read them. Logs and diagnostics now accumulate across steps. The graded output and error are still the last step's, so a successful repair step clears the failure that preceded it, but an unexpected failure stops the run. "Unexpected" needs to be explicit, because repair scenarios deliberately call a helper wrong and fix it from the structured error — that first failure is the whole point. `CodeModeEvalExecuteStep.expectsFailure` marks those, and `filesystemRepairAfterInvalidArguments` moves to explicit steps to use it. Every other step failing now ends the scenario. --- Sources/CodeModeEvaluation/EvalModels.swift | 26 ++++++++- Sources/CodeModeEvaluation/EvalRunner.swift | 23 ++++++-- .../CodeModeEvaluation/EvalScenarios.swift | 12 +++- .../CodeModeEvalRunnerTests.swift | 55 +++++++++++++++++++ 4 files changed, 108 insertions(+), 8 deletions(-) diff --git a/Sources/CodeModeEvaluation/EvalModels.swift b/Sources/CodeModeEvaluation/EvalModels.swift index b8ae289..5511a4b 100644 --- a/Sources/CodeModeEvaluation/EvalModels.swift +++ b/Sources/CodeModeEvaluation/EvalModels.swift @@ -54,15 +54,39 @@ public struct CodeModeEvalExecuteStep: Codable, Sendable, Equatable { public var code: String public var allowedCapabilities: [CapabilityID] public var timeoutMs: Int? + /// Whether this step is *supposed* to fail. + /// + /// Repair scenarios deliberately call a helper wrong and then fix it from the + /// structured error, so their first step failing is the point. Every other + /// step failing should stop the run — otherwise a later step masks it and the + /// scenario passes on a broken transcript. + public var expectsFailure: Bool public init( code: String, allowedCapabilities: [CapabilityID] = [], - timeoutMs: Int? = nil + timeoutMs: Int? = nil, + expectsFailure: Bool = false ) { self.code = code self.allowedCapabilities = allowedCapabilities self.timeoutMs = timeoutMs + self.expectsFailure = expectsFailure + } + + private enum CodingKeys: String, CodingKey { + case code + case allowedCapabilities + case timeoutMs + case expectsFailure + } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + self.code = try container.decode(String.self, forKey: .code) + self.allowedCapabilities = try container.decodeIfPresent([CapabilityID].self, forKey: .allowedCapabilities) ?? [] + self.timeoutMs = try container.decodeIfPresent(Int.self, forKey: .timeoutMs) + self.expectsFailure = try container.decodeIfPresent(Bool.self, forKey: .expectsFailure) ?? false } } diff --git a/Sources/CodeModeEvaluation/EvalRunner.swift b/Sources/CodeModeEvaluation/EvalRunner.swift index 30fba8a..2204a7f 100644 --- a/Sources/CodeModeEvaluation/EvalRunner.swift +++ b/Sources/CodeModeEvaluation/EvalRunner.swift @@ -58,6 +58,11 @@ public final class CodeModeEvalRunner: Sendable { searchDiagnostics = response.diagnostics } + // Logs and diagnostics accumulate across steps and an intermediate + // error stops the run. Overwriting each field per step meant + // validation only ever saw the last step: a scenario could pass while + // step 1 failed in a way step 2 masked, and step 1's logs — the + // evidence for why — were discarded. for executeStep in executeSteps(for: scenario) { toolCalls.append( CodeModeEvalToolCall( @@ -74,15 +79,25 @@ public final class CodeModeEvalRunner: Sendable { ) ) let observed = await observe(call) + // The graded value is the last step's output, and the graded error + // is the last step's error — so a successful repair step clears the + // deliberate failure that preceded it. executionOutput = observed.output - executionLogs = observed.logs - executionDiagnostics = observed.diagnostics observedError = observed.error + executionLogs.append(contentsOf: observed.logs) + executionDiagnostics.append(contentsOf: observed.diagnostics) + + // A repair scenario's first step is meant to fail and the next step + // fixes it. Anything else stops here rather than letting a later + // step mask the failure. + if observed.error != nil, executeStep.expectsFailure == false { + break + } } } catch let error as CodeModeToolError { observedError = error - executionLogs = error.logs - executionDiagnostics = error.diagnostics + executionLogs.append(contentsOf: error.logs) + executionDiagnostics.append(contentsOf: error.diagnostics) } catch { failures.append("Unexpected runner failure: \(error.localizedDescription)") } diff --git a/Sources/CodeModeEvaluation/EvalScenarios.swift b/Sources/CodeModeEvaluation/EvalScenarios.swift index 091e965..455b624 100644 --- a/Sources/CodeModeEvaluation/EvalScenarios.swift +++ b/Sources/CodeModeEvaluation/EvalScenarios.swift @@ -319,10 +319,16 @@ public enum CodeModeEvalScenarios { return api.byJSName["apple.fs.read"]; } """, - executeCode: """ - return await apple.fs.read({ encoding: "utf8" }); - """, executeSteps: [ + // Deliberately wrong, so the next step can repair from the structured + // error. Marked so the runner does not treat it as the run failing. + CodeModeEvalExecuteStep( + code: """ + return await apple.fs.read({ encoding: "utf8" }); + """, + allowedCapabilities: [.fsRead], + expectsFailure: true + ), CodeModeEvalExecuteStep( code: """ const result = await apple.fs.read({ path: "tmp:repair.txt", encoding: "utf8" }); diff --git a/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift b/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift index 340f01b..269749c 100644 --- a/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift +++ b/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift @@ -142,3 +142,58 @@ private func failureSummary(_ results: [CodeModeEvalResult]) -> String { } .joined(separator: "\n") } + +// MARK: - Multi-step scenarios + +// The runner overwrote executionOutput/Logs/Diagnostics/observedError on every +// step and did not stop on an intermediate error, so validation saw only the last +// step: a scenario could pass while step 1 failed in a way step 2 masked, and +// step 1's logs — the evidence — were gone. + +@Test func multiStepScenariosAccumulateLogsAcrossSteps() async throws { + let scenario = CodeModeEvalScenario( + id: "multi-step-log-accumulation", + title: "Logs from every step survive", + task: "internal test", + executeSteps: [ + CodeModeEvalExecuteStep(code: "console.log('step-one-ran'); return 1;", allowedCapabilities: []), + CodeModeEvalExecuteStep(code: "console.log('step-two-ran'); return 2;", allowedCapabilities: []), + ], + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript, .executeJavaScript], + exactAllowedCapabilities: [], + requiredExecutionLogFragments: ["step-one-ran", "step-two-ran"], + expectedOutput: .number(2) + ) + ) + + let result = await CodeModeEvalRunner().run(scenario) + #expect(result.passed, "\(result.failures)") + #expect(result.executionLogs.contains { $0.message.contains("step-one-ran") }) + #expect(result.executionLogs.contains { $0.message.contains("step-two-ran") }) + #expect(result.executionOutput == .number(2)) +} + +@Test func anIntermediateStepFailureStopsTheRunAndIsReported() async throws { + let scenario = CodeModeEvalScenario( + id: "multi-step-intermediate-failure", + title: "A failing first step is not masked by a passing second", + task: "internal test", + executeSteps: [ + // Denied: the step declares no capabilities. + CodeModeEvalExecuteStep(code: "return await apple.fs.read({ path: 'tmp:nope.txt' });", allowedCapabilities: []), + CodeModeEvalExecuteStep(code: "return 'second step succeeded';", allowedCapabilities: []), + ], + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript], + exactAllowedCapabilities: [], + expectedErrorCode: "CAPABILITY_DENIED" + ) + ) + + let result = await CodeModeEvalRunner().run(scenario) + #expect(result.error?.code == "CAPABILITY_DENIED") + // The second step must not have run and must not have supplied the output. + #expect(result.toolCalls.filter { $0.tool == .executeJavaScript }.count == 1) + #expect(result.executionOutput != .string("second step succeeded")) +} From f4748cc5a408c987e4e5ef01047dda00efa32b8f Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 20:29:00 -0700 Subject: [PATCH 09/18] Record the decision to keep swift-syntax in the core target REVIEW.md P1 #12 asks the CodeMode target to drop its CodeModeMacros dependency by checking in the 117 expanded tool definitions. Declined, with reasoning in TODO.md: the cited cost is already measured and mitigated (prebuilt swift-syntax is default-on, and the version range is already widened for the one named conflict), the migration is one-way and strips @ToolParam hints off the property declarations, and it pulls against findings #11 and #13, which both want more generation from this metadata. --- TODO.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/TODO.md b/TODO.md index 282f76a..0392115 100644 --- a/TODO.md +++ b/TODO.md @@ -238,6 +238,24 @@ tool model). → "resource- and network-bounded runtime." 3. Metadata-consolidation refactor + macro decision, protected by new drift tests. +## DECLINED REVIEW FINDINGS +- **"Stop imposing swift-syntax on every consumer" (REVIEW.md P1 #12)** — + considered 2026-07-25 and declined. The proposal is to check in the expanded + `@BuiltInCodeMode` members so the `CodeMode` target can drop its + `CodeModeMacros` dependency. Declined because: + - The cost it cites is already measured and mitigated: prebuilt swift-syntax is + default-on for macros on the supported toolchains (`PLAN-registration-macros.md` + §"The swift-syntax cost, measured today"), so consumers pay ~0 incremental + build time, and the version range is already widened to `602.0.0..<604.0.0` + for `mlx-swift-lm` co-resolution — the one concrete conflict named. + - The migration is effectively one-way across 117 tool definitions and strips + the `@ToolParam("hint")` annotations off the property declarations, leaving + the hints readable only inside the generated argument arrays. + - It works against REVIEW #11 and #13, which both ask for *more* generation + from this metadata, not less. + Revisit if a consumer hits a swift-syntax version conflict the range cannot + absorb, or if prebuilts stop being default-on. + ## OPEN QUESTIONS FOR THE USER - [ ] Decision on the private-but-exported JSC symbol (App Store risk)? - [ ] After CI is green: tag `0.1.0` and restore the versioned install snippet? From 1ef45cf72ab19f062db564c378f3dbd4d7b1162d Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 20:36:05 -0700 Subject: [PATCH 10/18] Generate TypeScript declarations from the capability registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloudflare's central Code Mode insight is that models write markedly better code against real types, and nothing here emitted any: the model got a flat `argumentTypes` map of six coarse cases, a prose `resultSummary`, and constraints that lived only in hint text it had to notice and obey. The registry already holds names, required/optional arguments, types, enum constraints, and hints, so `TypeScriptDeclarations` emits `.d.ts` from exactly that. Constrained arguments become string-literal unions — the constraint *is* the type. Dotted argument paths (`options.timeoutMs`) rebuild into nested object types. Hints become doc comments; capability ID, result summary, aliases, and the example ride along. Two entry points: - `JavaScriptAPIReference.dts` — per capability, so code-driven search stays the filter and TypeScript becomes the payload. The tool description now points the model at it. - `CodeModeAgentTools.typeDeclarations()` — the whole platform-filtered surface, namespaced, for a host system prompt. `capabilities()` is exposed alongside it, which hosts also needed for consent UI and for building a `CapabilityGrant`. Results are typed `CodeModeValue`: built-in bridges return untyped JSON dictionaries, so a narrower result type would be a fiction. Per-capability result schemas can tighten this later without changing the shape. Generating declarations surfaced real catalog-vs-binding drift, which is fixed rather than described (partial #11 — the hand-written bootstrap stays for now): - `apple.keychain.{get,set,delete}` advertised object arguments while the wrapper took positional strings, and the catalog's own example showed the positional form. Code written from the metadata sent "[object Object]" as the key. The wrappers accept both spellings; examples move to the object form. - `fs.promises.rename`/`copyFile` could not pass the `overwrite`/`recursive` flags that `fs.move`/`fs.copy` now require, so those aliases failed on any existing destination. - `apple.location.getCurrentPosition`/`getPermissionStatus` discarded a caller's arguments outright; they now default the mode instead of forcing it. - The tool description claimed "apple.* helpers take one object argument" as a blanket rule; it now scopes positional to `fs.promises.*` and points at `dts`. The Node-compatibility globals stay in a hand-authored preamble, because their positional convention is not something the catalog can express — that gap is exactly what #11 exists to close. --- README.md | 34 ++ Sources/CodeMode/API/BridgeModels.swift | 8 +- .../SimpleBuiltInCodeModeProviders.swift | 6 +- Sources/CodeMode/Catalog/BridgeCatalog.swift | 11 +- .../Catalog/TypeScriptDeclarations.swift | 343 ++++++++++++++++++ .../Host/CodeModeAgentToolDescriptions.swift | 14 +- .../CodeMode/Host/CodeModeAgentTools.swift | 19 + .../CodeMode/Runtime/RuntimeJavaScript.swift | 45 ++- .../TypeScriptDeclarationsTests.swift | 199 ++++++++++ .../capability-metadata-golden.json | 6 +- 10 files changed, 668 insertions(+), 17 deletions(-) create mode 100644 Sources/CodeMode/Catalog/TypeScriptDeclarations.swift create mode 100644 Tests/CodeModeTests/TypeScriptDeclarationsTests.swift diff --git a/README.md b/README.md index d8a45a0..6624c81 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,40 @@ let tools = CodeModeAgentTools( ) ``` +## TypeScript Declarations + +The registry already knows argument names, types, optionality, enum constraints, +and hints. That metadata is emitted as TypeScript so the model writes code +against real declarations instead of a coarse type map plus prose: + +```swift +let tools = CodeModeAgentTools() +print(tools.typeDeclarations()) // whole platform-filtered surface +print(tools.capabilities()) // every reference, each carrying `dts` +``` + +Every `JavaScriptAPIReference` carries a per-capability `dts`, so code-driven +search stays the filter and TypeScript becomes the payload: + +```javascript +async () => { + return api.references + .filter(ref => ref.tags.includes("calendar")) + .map(ref => ref.dts) + .join("\n\n"); +} +``` + +Constrained arguments become string-literal unions, dotted argument paths +(`options.timeoutMs`) become nested object types, and argument hints become doc +comments. Results are typed `CodeModeValue` — built-in bridges return untyped +JSON dictionaries, so a narrower result type would be a fiction; the prose result +summary is in the doc comment until per-capability result schemas exist. + +The Node-compatibility globals (`fetch`, `fs.promises.*`, `path`, `console`) are +declared in a hand-authored preamble, because their positional calling convention +is not something the catalog can express. + ## Search `searchJavaScriptAPI` accepts `JavaScriptAPISearchRequest`: diff --git a/Sources/CodeMode/API/BridgeModels.swift b/Sources/CodeMode/API/BridgeModels.swift index 3e282e3..d86b492 100644 --- a/Sources/CodeMode/API/BridgeModels.swift +++ b/Sources/CodeMode/API/BridgeModels.swift @@ -102,6 +102,10 @@ public struct JavaScriptAPIReference: Sendable, Codable, Equatable { public var argumentHints: [String: String] public var argumentConstraints: CapabilityArgumentConstraints public var resultSummary: String + /// TypeScript declaration for this helper, generated from the fields above. + /// Return it from `searchJavaScriptAPI` to give the model real types rather + /// than a coarse `argumentTypes` map plus prose. + public var dts: String public init( capability: String, @@ -116,7 +120,8 @@ public struct JavaScriptAPIReference: Sendable, Codable, Equatable { argumentTypes: [String: CapabilityArgumentType], argumentHints: [String: String], argumentConstraints: CapabilityArgumentConstraints = .none, - resultSummary: String + resultSummary: String, + dts: String = "" ) { self.capability = capability self.capabilityKey = capabilityKey ?? CodeModeCapabilityKey(rawValue: capability) @@ -131,6 +136,7 @@ public struct JavaScriptAPIReference: Sendable, Codable, Equatable { self.argumentHints = argumentHints self.argumentConstraints = argumentConstraints self.resultSummary = resultSummary + self.dts = dts } } diff --git a/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift b/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift index 194d853..0066ce5 100644 --- a/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift +++ b/Sources/CodeMode/Bridges/SimpleBuiltInCodeModeProviders.swift @@ -39,7 +39,7 @@ private struct KeychainReadTool: BuiltInCodeModeTool { static let codeModeTitle = "Read Keychain value" static let codeModeSummary = "Read a string value from app-scoped Keychain storage." static let codeModeTags = ["security", "token", "keychain"] - static let codeModeExample = "await apple.keychain.get('auth_token')" + static let codeModeExample = "await apple.keychain.get({ key: 'auth_token' })" static let codeModeArguments = [ BuiltInToolArgument("key", .string, hint: "Logical key for this secret value."), ] @@ -67,7 +67,7 @@ private struct KeychainWriteTool: BuiltInCodeModeTool { static let codeModeTitle = "Write Keychain value" static let codeModeSummary = "Store or update a string value in app-scoped Keychain storage." static let codeModeTags = ["security", "token", "keychain"] - static let codeModeExample = "await apple.keychain.set('auth_token', token)" + static let codeModeExample = "await apple.keychain.set({ key: 'auth_token', value: token })" static let codeModeArguments = [ BuiltInToolArgument("key", .string, hint: "Logical key for this secret value."), BuiltInToolArgument("value", .string, optional: true, hint: "Secret string value. Defaults to empty string when omitted."), @@ -102,7 +102,7 @@ private struct KeychainDeleteTool: BuiltInCodeModeTool { static let codeModeTitle = "Delete Keychain value" static let codeModeSummary = "Delete an app-scoped Keychain value." static let codeModeTags = ["security", "token", "keychain"] - static let codeModeExample = "await apple.keychain.delete('auth_token')" + static let codeModeExample = "await apple.keychain.delete({ key: 'auth_token' })" static let codeModeArguments = [ BuiltInToolArgument("key", .string, hint: "Logical key for value removal."), ] diff --git a/Sources/CodeMode/Catalog/BridgeCatalog.swift b/Sources/CodeMode/Catalog/BridgeCatalog.swift index 9e55914..6e65647 100644 --- a/Sources/CodeMode/Catalog/BridgeCatalog.swift +++ b/Sources/CodeMode/Catalog/BridgeCatalog.swift @@ -52,6 +52,12 @@ struct BridgeCatalog: Sendable { searchCatalog } + /// TypeScript declarations for the whole platform-filtered surface, for a + /// host to drop into its system prompt. + func typeDeclarations() -> String { + TypeScriptDeclarations.surface(for: references) + } + func closestFunctionNames(to candidate: String, limit: Int = 3) -> [String] { let normalizedCandidate = candidate .lowercased() @@ -99,7 +105,7 @@ struct BridgeCatalog: Sendable { } private static func reference(from function: RegisteredCodeModeFunction) -> JavaScriptAPIReference { - return JavaScriptAPIReference( + let reference = JavaScriptAPIReference( capability: function.catalogCapability, capabilityKey: function.capabilityKey, builtInCapability: function.builtInCapability, @@ -114,6 +120,9 @@ struct BridgeCatalog: Sendable { argumentConstraints: function.argumentConstraints, resultSummary: function.resultSummary ) + var withTypes = reference + withTypes.dts = TypeScriptDeclarations.declaration(for: reference) + return withTypes } private static func levenshtein(_ lhs: String, _ rhs: String) -> Int { diff --git a/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift b/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift new file mode 100644 index 0000000..81e891d --- /dev/null +++ b/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift @@ -0,0 +1,343 @@ +import Foundation + +/// Generates TypeScript declarations for the CodeMode JavaScript surface. +/// +/// The registry already holds names, required/optional arguments, types, enum +/// constraints, and hints; before this, the model saw that as a flat +/// `argumentTypes` map of six coarse cases and a prose `resultSummary`. Models +/// write markedly better code against real declarations, so the same metadata is +/// emitted as `.d.ts` — per capability for search results, and whole-surface for +/// a system prompt. +/// +/// Result types are `CodeModeValue` for now: built-in tools return untyped +/// `JSONValue` dictionaries, so a typed result would be a fiction. The prose +/// `resultSummary` is carried in the doc comment instead, and a per-capability +/// result schema can tighten this later without changing the shape here. +public enum TypeScriptDeclarations { + /// Shared types and the runtime globals that are not catalog capabilities. + /// + /// Hand-authored deliberately: these are the Node-compatibility shims in + /// `RuntimeJavaScript.bootstrap`, and unlike the namespaced helpers they take + /// *positional* arguments, which the catalog has no way to express. Keeping + /// them here means the declarations tell the truth about both conventions. + public static let preamble = """ + /** Any JSON-representable value. Bridge results are JSON, not class instances. */ + type CodeModeValue = + | null + | boolean + | number + | string + | CodeModeValue[] + | { [key: string]: CodeModeValue }; + + declare const console: { + log(...values: unknown[]): void; + info(...values: unknown[]): void; + warn(...values: unknown[]): void; + error(...values: unknown[]): void; + }; + + interface CodeModeResponse { + ok: boolean; + status: number; + statusText: string; + headers: { get(name: string): string | null }; + text(): Promise; + json(): Promise; + } + + /** + * HTTP(S) fetch. Runs on an isolated session: no app cookies, no stored + * credentials. Cookie/Authorization/Proxy-* request headers are refused + * unless the host app permits them. + */ + declare function fetch( + url: string, + options?: { + method?: string; + headers?: { [name: string]: string }; + body?: string; + bodyBase64?: string; + timeoutMs?: number; + responseEncoding?: "text" | "base64"; + } + ): Promise; + + /** + * Node-style filesystem aliases. Unlike the apple.* helpers these take + * POSITIONAL arguments and return Node-like values. + */ + declare const fs: { + promises: { + readFile(path: string, encoding?: "utf8" | "base64"): Promise; + writeFile(path: string, data: string, encoding?: "utf8" | "base64"): Promise; + readdir(path: string): Promise; + stat(path: string): Promise; + access(path: string): Promise; + mkdir(path: string, options?: { recursive?: boolean }): Promise; + rm(path: string, options?: { recursive?: boolean }): Promise; + rename(from: string, to: string, options?: { overwrite?: boolean; recursive?: boolean }): Promise; + copyFile(from: string, to: string, options?: { overwrite?: boolean; recursive?: boolean }): Promise; + }; + }; + + declare const path: { + join(...parts: string[]): string; + }; + """ + + /// The declaration for one capability, suitable for a search result payload. + public static func declaration(for reference: JavaScriptAPIReference) -> String { + guard let canonical = canonicalName(for: reference) else { + return "" + } + let signature = functionSignature(for: reference, name: lastComponent(of: canonical)) + return "\(documentation(for: reference, canonicalName: canonical))\ndeclare function \(canonical.replacingOccurrences(of: ".", with: "_"))\(signature);" + } + + /// The whole surface: the preamble plus every capability, grouped into + /// namespaces by its canonical dotted JavaScript name. + public static func surface(for references: [JavaScriptAPIReference]) -> String { + var root = Namespace(name: "") + for reference in references.sorted(by: { $0.capability < $1.capability }) { + guard let canonical = canonicalName(for: reference) else { + continue + } + var components = canonical.split(separator: ".").map(String.init) + guard components.count >= 2 else { + // A bare global such as `fetch` is covered by the preamble. + continue + } + let functionName = components.removeLast() + root.insert( + Member( + name: functionName, + documentation: documentation(for: reference, canonicalName: canonical), + signature: functionSignature(for: reference, name: functionName) + ), + at: components + ) + } + + let body = root.children + .sorted { $0.key < $1.key } + .map { render(namespace: $0.value, indent: "", isTopLevel: true) } + .joined(separator: "\n\n") + + return """ + // CodeMode JavaScript API — generated from the capability registry. + // Only helpers supported on the current host platform appear here. + + \(preamble) + + \(body) + """ + } + + // MARK: - Declaration pieces + + /// The namespaced helper name, preferred over Node-style aliases: aliases use + /// positional arguments the catalog cannot describe, and are documented in + /// the preamble instead. + private static func canonicalName(for reference: JavaScriptAPIReference) -> String? { + reference.jsNames.first { $0.contains(".") && $0.hasPrefix("fs.promises.") == false } + ?? reference.jsNames.first + } + + private static func lastComponent(of name: String) -> String { + name.split(separator: ".").last.map(String.init) ?? name + } + + private static func documentation(for reference: JavaScriptAPIReference, canonicalName: String) -> String { + var lines = ["/**"] + lines.append(" * \(reference.summary)") + lines.append(" *") + lines.append(" * Capability: \(reference.capability)") + lines.append(" * Returns: \(reference.resultSummary)") + + let aliases = reference.jsNames.filter { $0 != canonicalName } + if aliases.isEmpty == false { + // Only the fs.promises.* shims are positional; other aliases are + // additional namespaced spellings of the same object-argument call. + let positional = aliases.filter { $0.hasPrefix("fs.promises.") || $0 == "fetch" } + let note = positional.isEmpty ? "" : " — \(positional.joined(separator: ", ")) take positional arguments" + lines.append(" * Aliases: \(aliases.joined(separator: ", "))\(note)") + } + if reference.example.isEmpty == false { + lines.append(" *") + lines.append(" * @example \(reference.example)") + } + lines.append(" */") + return lines.joined(separator: "\n") + } + + private static func functionSignature(for reference: JavaScriptAPIReference, name: String) -> String { + let tree = ArgumentTree(reference: reference) + guard tree.isEmpty == false else { + return "(): Promise" + } + let hasRequired = reference.requiredArguments.isEmpty == false + return "(args\(hasRequired ? "" : "?"): \(tree.render(indent: "")))" + ": Promise" + } + + // MARK: - Argument tree + // + // Arguments are declared as dotted paths (`options.timeoutMs`), so the object + // literal has to be rebuilt from them. + + private struct ArgumentTree { + private var children: [String: ArgumentTree] = [:] + private var leaves: [Leaf] = [] + + private struct Leaf { + var name: String + var type: String + var optional: Bool + var hint: String? + } + + init(reference: JavaScriptAPIReference) { + let allPaths = Array(Set( + reference.requiredArguments + reference.optionalArguments + Array(reference.argumentTypes.keys) + )).sorted() + let required = Set(reference.requiredArguments) + + for path in allPaths { + let components = path.split(separator: ".").map(String.init) + insert( + components: components, + leaf: Leaf( + name: components.last ?? path, + type: TypeScriptDeclarations.tsType( + for: reference.argumentTypes[path] ?? .any, + allowedValues: reference.argumentConstraints.allowedStringValues[path] + ), + // A nested path is optional unless the exact path is + // listed as required. + optional: required.contains(path) == false, + hint: reference.argumentHints[path] + ) + ) + } + } + + private init() {} + + var isEmpty: Bool { + children.isEmpty && leaves.isEmpty + } + + private mutating func insert(components: [String], leaf: Leaf) { + guard components.count > 1 else { + // A declared parent (`options: object`) whose children also appear + // is represented by the nested literal, not by a duplicate leaf. + if children[leaf.name] == nil, leaves.contains(where: { $0.name == leaf.name }) == false { + leaves.append(leaf) + } + return + } + let head = components[0] + leaves.removeAll { $0.name == head } + var child = children[head] ?? ArgumentTree() + child.insert(components: Array(components.dropFirst()), leaf: leaf) + children[head] = child + } + + func render(indent: String) -> String { + let inner = indent + " " + var lines: [String] = ["{"] + + for leaf in leaves.sorted(by: { $0.name < $1.name }) { + if let hint = leaf.hint, hint.isEmpty == false { + lines.append("\(inner)/** \(hint) */") + } + lines.append("\(inner)\(leaf.name)\(leaf.optional ? "?" : ""): \(leaf.type);") + } + + for (name, child) in children.sorted(by: { $0.key < $1.key }) { + lines.append("\(inner)\(name)?: \(child.render(indent: inner));") + } + + lines.append("\(indent)}") + return lines.joined(separator: "\n") + } + } + + private static func tsType(for type: CapabilityArgumentType, allowedValues: [String]?) -> String { + if let allowedValues, allowedValues.isEmpty == false { + // The constraint is the type: a union beats `string` plus prose the + // model has to notice. + return allowedValues.map { "\"\($0)\"" }.joined(separator: " | ") + } + + switch type { + case .string: + return "string" + case .number: + return "number" + case .bool: + return "boolean" + case .object: + return "{ [key: string]: CodeModeValue }" + case .array: + return "CodeModeValue[]" + case .any: + return "CodeModeValue" + } + } + + // MARK: - Namespace rendering + + private struct Member { + var name: String + var documentation: String + var signature: String + } + + private final class Namespace { + let name: String + var children: [String: Namespace] = [:] + var members: [Member] = [] + + init(name: String) { + self.name = name + } + + func insert(_ member: Member, at path: [String]) { + guard let head = path.first else { + members.append(member) + return + } + let child = children[head] ?? Namespace(name: head) + children[head] = child + child.insert(member, at: Array(path.dropFirst())) + } + } + + private static func render(namespace: Namespace, indent: String, isTopLevel: Bool) -> String { + let inner = indent + " " + var lines = ["\(indent)\(isTopLevel ? "declare namespace" : "namespace") \(namespace.name) {"] + + for member in namespace.members.sorted(by: { $0.name < $1.name }) { + lines.append(indented(member.documentation, by: inner)) + lines.append("\(inner)function \(member.name)\(indented(member.signature, by: inner, skipFirstLine: true));") + } + + for (_, child) in namespace.children.sorted(by: { $0.key < $1.key }) { + lines.append(render(namespace: child, indent: inner, isTopLevel: false)) + } + + lines.append("\(indent)}") + return lines.joined(separator: "\n") + } + + private static func indented(_ text: String, by indent: String, skipFirstLine: Bool = false) -> String { + text + .split(separator: "\n", omittingEmptySubsequences: false) + .enumerated() + .map { index, line in + index == 0 && skipFirstLine ? String(line) : indent + line + } + .joined(separator: "\n") + } +} diff --git a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift index 428ca7d..50fb554 100644 --- a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift +++ b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift @@ -75,6 +75,7 @@ public enum CodeModeAgentToolDescriptions { argumentHints: Record; argumentConstraints: { allowedStringValues: Record }; resultSummary: string; + dts: string; } declare const api: { @@ -86,6 +87,8 @@ public enum CodeModeAgentToolDescriptions { Your code must evaluate to an async function and return JSON-serializable output. Search has a 2s budget, so slice broad result sets before returning them. + Prefer returning ref.dts: it is the TypeScript declaration for the helper, with argument names, types, string-literal unions for constrained values, optionality, and the result summary. It is more precise and usually shorter than assembling argumentTypes, argumentHints, and argumentConstraints yourself. + Examples: async () => { return api.references @@ -106,7 +109,14 @@ public enum CodeModeAgentToolDescriptions { } async () => { - return api.byJSName["apple.fs.read"]; + return api.byJSName["apple.fs.read"].dts; + } + + async () => { + return api.references + .filter(ref => ref.tags.includes("calendar")) + .map(ref => ref.dts) + .join("\n\n"); } """ ) @@ -116,7 +126,7 @@ public enum CodeModeAgentToolDescriptions { description: """ Execute JavaScript against the CodeMode runtime. Prefer searchJavaScriptAPI first when choosing helpers or arguments. Cross-platform helpers live under apple.* and platform-specific helpers live under platform namespaces such as ios.alarm.*; custom host providers may expose additional namespaces. System UI helpers such as apple.ui.presentAlert, apple.calendar.presentNewEvent, apple.photos.pick, apple.contacts.pick, apple.documents.pick, apple.share.present, apple.quicklook.preview, apple.web.present, and apple.auth.webAuthenticate require a host-provided SystemUIPresenter; camera, document scan, mail, and message compose helpers are iOS-only. Only helpers supported on the current host platform are installed. - Calling conventions: apple.* helpers take one object argument, for example apple.fs.read({ path: "tmp:file.txt" }) and often return structured objects such as { text, base64 }. Node-style filesystem aliases use positional arguments, for example fs.promises.readFile("tmp:file.txt", "utf8"), and return Node-like values such as a string for readFile. fetch(url, options) is a global helper and returns a Response-like object with text(), json(), headers.get(), status, and ok. + Calling conventions: apple.* and other namespaced helpers take one object argument, for example apple.fs.read({ path: "tmp:file.txt" }) and often return structured objects such as { text, base64 }. Node-style filesystem aliases under fs.promises.* use positional arguments, for example fs.promises.readFile("tmp:file.txt", "utf8"), and return Node-like values such as a string for readFile. When in doubt, use the ref.dts from searchJavaScriptAPI — it states the exact signature. fetch(url, options) is a global helper and returns a Response-like object with text(), json(), headers.get(), status, and ok. Return semantics: the runtime wraps your code in an async function. For multi-statement code, return the final graded value with an explicit top-level return statement. A script that is only a bare final top-level await expression, such as await apple.fs.read({ path: "tmp:file.txt" }), returns that awaited value. Do not use an unreturned async IIFE as the final expression. setTimeout callbacks fire synchronously in this runtime. diff --git a/Sources/CodeMode/Host/CodeModeAgentTools.swift b/Sources/CodeMode/Host/CodeModeAgentTools.swift index a269604..5fb7f7f 100644 --- a/Sources/CodeMode/Host/CodeModeAgentTools.swift +++ b/Sources/CodeMode/Host/CodeModeAgentTools.swift @@ -46,7 +46,26 @@ public final class CodeModeAgentTools: @unchecked Sendable { try await runtime.searchAsync(request) } + /// Declared `async throws` for a body that is currently synchronous and + /// non-throwing. That is deliberate: making the bridge ABI asynchronous is + /// planned, and this signature is the one that will not have to change. public func executeJavaScript(_ request: JavaScriptExecutionRequest) async throws -> JavaScriptExecutionCall { runtime.makeExecutionCall(request) } + + /// TypeScript declarations for the whole platform-filtered API surface. + /// + /// Models write markedly better code against real types than against prose, + /// so a host can drop this into its system prompt when the surface is small + /// enough to afford, and rely on `searchJavaScriptAPI` (whose results carry a + /// per-capability `dts`) when it is not. + public func typeDeclarations() -> String { + catalog.typeDeclarations() + } + + /// Every capability available on this host, for consent UI and for hosts + /// building their own `CapabilityGrant`. + public func capabilities() -> [JavaScriptAPIReference] { + catalog.allReferences() + } } diff --git a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift index f10ef84..45d2855 100644 --- a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift +++ b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift @@ -258,16 +258,32 @@ enum RuntimeJavaScript { }; globalThis.apple = globalThis.apple || {}; + // These accept either the object form the catalog advertises — + // apple.keychain.get({ key }) — or the positional form the examples have + // always shown. The catalog described object arguments while the wrapper took + // a positional string, so code written from the metadata sent "[object + // Object]" as the key. + function __keychainArgs(first, value) { + if (first && typeof first === 'object') { + const args = { key: String(first.key) }; + if (first.value !== undefined) args.value = String(first.value); + return args; + } + const args = { key: String(first) }; + if (value !== undefined) args.value = String(value); + return args; + } + globalThis.apple.keychain = { - get: function(key) { return __invokeAsync('keychain.read', { key: String(key) }); }, - set: function(key, value) { return __invokeAsync('keychain.write', { key: String(key), value: String(value) }); }, - delete: function(key) { return __invokeAsync('keychain.delete', { key: String(key) }); } + get: function(key) { return __invokeAsync('keychain.read', __keychainArgs(key)); }, + set: function(key, value) { return __invokeAsync('keychain.write', __keychainArgs(key, value)); }, + delete: function(key) { return __invokeAsync('keychain.delete', __keychainArgs(key)); } }; globalThis.apple.location = { - getPermissionStatus: function() { return __invokeAsync('location.read', { mode: 'permissionStatus' }); }, + getPermissionStatus: function(args) { return __invokeAsync('location.read', Object.assign({ mode: 'permissionStatus' }, args || {})); }, requestPermission: function() { return __invokeAsync('location.permission.request', {}); }, - getCurrentPosition: function() { return __invokeAsync('location.read', { mode: 'current' }); } + getCurrentPosition: function(args) { return __invokeAsync('location.read', Object.assign({ mode: 'current' }, args || {})); } }; globalThis.apple.calendar = { @@ -353,6 +369,17 @@ enum RuntimeJavaScript { open: function(args) { return __invokeAsync('settings.ui.open', args || {}); } }; + // fs.move / fs.copy refuse an existing destination without an explicit + // overwrite, so the Node-style aliases need a way to pass one through. + function __fsDestinationArgs(from, to, options) { + const args = { from: String(from), to: String(to) }; + if (options && typeof options === 'object') { + if (options.overwrite !== undefined) args.overwrite = !!options.overwrite; + if (options.recursive !== undefined) args.recursive = !!options.recursive; + } + return args; + } + globalThis.fs = { promises: { readFile: function(path, options) { @@ -378,8 +405,12 @@ enum RuntimeJavaScript { rm: function(path, options) { return __invokeAsync('fs.delete', { path: String(path), recursive: !!(options && options.recursive) }); }, - rename: function(from, to) { return __invokeAsync('fs.move', { from: String(from), to: String(to) }); }, - copyFile: function(from, to) { return __invokeAsync('fs.copy', { from: String(from), to: String(to) }); } + rename: function(from, to, options) { + return __invokeAsync('fs.move', __fsDestinationArgs(from, to, options)); + }, + copyFile: function(from, to, options) { + return __invokeAsync('fs.copy', __fsDestinationArgs(from, to, options)); + } } }; diff --git a/Tests/CodeModeTests/TypeScriptDeclarationsTests.swift b/Tests/CodeModeTests/TypeScriptDeclarationsTests.swift new file mode 100644 index 0000000..4421ea2 --- /dev/null +++ b/Tests/CodeModeTests/TypeScriptDeclarationsTests.swift @@ -0,0 +1,199 @@ +import Foundation +import Testing +@testable import CodeMode + +// Cloudflare's central Code Mode insight is that models write markedly better +// code against real types. Before this the model got a flat `argumentTypes` map +// of six coarse cases plus a prose `resultSummary`; these tests pin the shape of +// what it gets now. + +private func reference( + capability: String = "fs.read", + jsNames: [String] = ["apple.fs.read", "fs.promises.readFile"], + required: [String] = ["path"], + optional: [String] = ["encoding"], + types: [String: CapabilityArgumentType] = ["path": .string, "encoding": .string], + hints: [String: String] = ["path": "File path using allowed root prefix."], + constraints: CapabilityArgumentConstraints = .init(allowedStringValues: ["encoding": ["utf8", "base64"]]) +) -> JavaScriptAPIReference { + JavaScriptAPIReference( + capability: capability, + jsNames: jsNames, + summary: "Read a file.", + tags: ["fs"], + example: "await apple.fs.read({ path: 'tmp:x.txt' })", + requiredArguments: required, + optionalArguments: optional, + argumentTypes: types, + argumentHints: hints, + argumentConstraints: constraints, + resultSummary: "Object with path plus text or base64." + ) +} + +@Test func declarationsTypeArgumentsAndCarryHintsAsDocComments() { + let dts = TypeScriptDeclarations.declaration(for: reference()) + + #expect(dts.contains("path: string;")) + #expect(dts.contains("/** File path using allowed root prefix. */")) + // Required vs optional is expressed in the type, not left to prose. + #expect(dts.contains("encoding?:")) + #expect(dts.contains("path?:") == false) + #expect(dts.contains("Capability: fs.read")) + #expect(dts.contains("Returns: Object with path plus text or base64.")) +} + +@Test func constrainedArgumentsBecomeStringLiteralUnions() { + let dts = TypeScriptDeclarations.declaration(for: reference()) + // The constraint IS the type; a union beats `string` plus a prose hint the + // model has to notice and obey. + #expect(dts.contains(#"encoding?: "utf8" | "base64";"#)) +} + +@Test func nodeStyleAliasesAreDocumentedRatherThanMistypedAsObjectCalls() { + let dts = TypeScriptDeclarations.declaration(for: reference()) + // `fs.promises.readFile` is positional, which the catalog cannot express, so + // the generated signature must describe the namespaced helper and only + // mention the alias. + #expect(dts.contains("Aliases: fs.promises.readFile")) + #expect(dts.contains("positional")) +} + +@Test func dottedArgumentPathsBecomeNestedObjectTypes() { + let nested = reference( + capability: "network.fetch", + jsNames: ["fetch"], + required: ["url"], + optional: ["options.method", "options.timeoutMs"], + types: ["url": .string, "options.method": .string, "options.timeoutMs": .number], + hints: ["options.method": "HTTP method; defaults to GET."], + constraints: .none + ) + let dts = TypeScriptDeclarations.declaration(for: nested) + + #expect(dts.contains("options?: {")) + #expect(dts.contains("method?: string;")) + #expect(dts.contains("timeoutMs?: number;")) +} + +@Test func argumentlessHelpersTakeNoArgument() { + let none = reference( + capability: "location.permission.request", + jsNames: ["apple.location.requestPermission"], + required: [], + optional: [], + types: [:], + hints: [:], + constraints: .none + ) + #expect(TypeScriptDeclarations.declaration(for: none).contains("(): Promise")) +} + +@Test func theWholeSurfaceIsNamespacedAndPlatformFiltered() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .macOS) + defer { cleanup(sandbox) } + + let surface = tools.typeDeclarations() + + #expect(surface.contains("declare namespace apple {")) + #expect(surface.contains("namespace fs {")) + #expect(surface.contains("function read(args: {")) + // The Node-compat globals live in the hand-authored preamble, because their + // positional convention is not something the catalog can describe. + #expect(surface.contains("declare function fetch(")) + #expect(surface.contains("type CodeModeValue =")) + // iOS-only helpers must not be declared for a macOS host. + #expect(surface.contains("namespace alarm {") == false) +} + +@Test func iOSOnlyHelpersAppearForAniOSHost() async throws { + let (tools, sandbox) = try makeTools(hostPlatform: .iOS) + defer { cleanup(sandbox) } + #expect(tools.typeDeclarations().contains("namespace alarm {")) +} + +@Test func everyCapabilityCarriesItsOwnDeclarationForSearchResults() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let references = tools.capabilities() + #expect(references.isEmpty == false) + #expect(references.allSatisfy { $0.dts.isEmpty == false }) + + let read = try #require(references.first { $0.capability == "fs.read" }) + #expect(read.dts.contains("path: string;")) +} + +@Test func searchCanReturnDeclarationsAsItsPayload() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Code-driven search stays the filter; the payload is now TypeScript. + let response = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest( + code: """ + async () => { + return api.references + .filter(ref => ref.tags.includes("filesystem")) + .map(ref => ref.dts) + .join("\\n\\n"); + } + """ + ) + ) + let payload = try #require(response.result?.stringValue) + #expect(payload.contains("function apple_fs_read(args: {")) + #expect(payload.contains("path: string;")) +} + +@Test func nodeAliasesCanPassTheOverwriteFlagTheDeclarationPromises() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // fs.move/fs.copy now refuse an existing destination without an explicit + // overwrite, so the Node-style aliases had to gain a way to pass one — the + // preamble declares it, and this proves the runtime honours it. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.fs.write({ path: 'tmp:src.txt', data: 'source' }); + await apple.fs.write({ path: 'tmp:dst.txt', data: 'destination' }); + await fs.promises.copyFile('tmp:src.txt', 'tmp:dst.txt', { overwrite: true }); + return await fs.promises.readFile('tmp:dst.txt', 'utf8'); + """, + allowedCapabilities: [.fsWrite, .fsRead, .fsCopy] + ) + ) + + #expect(observed.error == nil) + #expect(observed.result?.output == .string("source")) +} + +@Test func keychainHelpersAcceptTheObjectFormTheCatalogAdvertises() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // The catalog declared `key: string` as an object argument while the wrapper + // took a positional string, and its own example showed the positional form — + // so code written from the metadata sent "[object Object]" as the key. Both + // spellings now work, which is what makes the generated declaration true. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.keychain.set({ key: 'codemode-dts-test', value: 'object-form' }); + const viaObject = await apple.keychain.get({ key: 'codemode-dts-test' }); + const viaPositional = await apple.keychain.get('codemode-dts-test'); + await apple.keychain.delete({ key: 'codemode-dts-test' }); + return { viaObject: viaObject && viaObject.value, viaPositional: viaPositional && viaPositional.value }; + """, + allowedCapabilities: [.keychainRead, .keychainWrite, .keychainDelete] + ) + ) + + #expect(observed.error == nil) + let output = try #require(observed.result?.output?.objectValue) + #expect(output.string("viaObject") == "object-form") + #expect(output.string("viaPositional") == "object-form") +} diff --git a/Tests/CodeModeTests/capability-metadata-golden.json b/Tests/CodeModeTests/capability-metadata-golden.json index 8e94ec8..a48e9ab 100644 --- a/Tests/CodeModeTests/capability-metadata-golden.json +++ b/Tests/CodeModeTests/capability-metadata-golden.json @@ -2398,7 +2398,7 @@ "argumentTypes" : { "key" : "string" }, - "example" : "await apple.keychain.delete('auth_token')", + "example" : "await apple.keychain.delete({ key: 'auth_token' })", "id" : "keychain.delete", "jsNames" : [ "apple.keychain.delete" @@ -2431,7 +2431,7 @@ "argumentTypes" : { "key" : "string" }, - "example" : "await apple.keychain.get('auth_token')", + "example" : "await apple.keychain.get({ key: 'auth_token' })", "id" : "keychain.read", "jsNames" : [ "apple.keychain.get" @@ -2466,7 +2466,7 @@ "key" : "string", "value" : "string" }, - "example" : "await apple.keychain.set('auth_token', token)", + "example" : "await apple.keychain.set({ key: 'auth_token', value: token })", "id" : "keychain.write", "jsNames" : [ "apple.keychain.set" From afa3f0706719d07429406536ef7a987a846fefe0 Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 20:42:20 -0700 Subject: [PATCH 11/18] Give the runtime a real timer loop and stop hiding swallowed failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Partial P1 #9. The full async bridge ABI — Swift-resolved promises, concurrent native calls, bounded execution slots — remains a milestone; these are the parts that are wrong *within* the synchronous model, including both cheap wins the review called out as available immediately. **setTimeout was a lie.** It invoked the callback synchronously and ignored the delay, so three things were wrong at once: `clearTimeout` could never cancel, an error thrown by a callback propagated to setTimeout's *caller* rather than being an uncaught task error, and `await new Promise(r => setTimeout(r, 2000))` — the standard backoff — became a hot loop hammering remote APIs through `fetch`. There is now a real timer queue drained by the host between JavaScript turns: callbacks are deferred, delays are honoured, registration order breaks ties, `clearTimeout` works, and a throwing callback becomes a `TIMER_CALLBACK_ERROR` diagnostic. Waiting stays responsive to `cancel()` and bounded by `timeoutMs`, so a 30s timer under a 100ms budget still times out at 100ms. **Unsettleable promises are diagnosed, not waited out.** `waitForSettlement` polled a state that, under this model, only a timer can change — microtasks have already drained inside `evaluateScript` and no native call is outstanding. With no timer queued the promise is *provably* unsettleable, so it reports `JS_RUNTIME_ERROR: ... can never settle` immediately instead of sleeping a thread to the deadline for a result that cannot arrive. **Swallowed bridge failures are visible.** Nothing installs an unhandled-rejection hook, so a forgotten `await` — the most common LLM JavaScript mistake — discarded a `CAPABILITY_DENIED` and the agent was told the run succeeded. A script that completes successfully despite failed bridge calls now carries a `BRIDGE_FAILURES_NOT_SURFACED` warning naming them. It is a warning, not an error, because a deliberate try/catch is indistinguishable from a missing await. Tool description and README updated: setTimeout is no longer described as synchronous, `Promise.all` is documented as sequential (which it is, until the async ABI lands), and the new diagnostics have repair guidance. The eval suite splits the old "unresolved promises time out" scenario into an unsettleable-promise case and a genuine CPU-bound timeout, and gains a timer-backoff scenario. --- README.md | 19 ++ .../Host/CodeModeAgentToolDescriptions.swift | 10 +- .../Runtime/BridgeInvocationContext.swift | 18 ++ Sources/CodeMode/Runtime/BridgeRuntime.swift | 141 ++++++++++- .../CodeMode/Runtime/RuntimeJavaScript.swift | 89 +++++-- .../CodeModeEvaluation/EvalScenarios.swift | 49 +++- Tests/CodeModeTests/EventLoopTests.swift | 224 ++++++++++++++++++ Tests/CodeModeTests/HostRuntimeTests.swift | 16 +- 8 files changed, 536 insertions(+), 30 deletions(-) create mode 100644 Tests/CodeModeTests/EventLoopTests.swift diff --git a/README.md b/README.md index 6624c81..210228c 100644 --- a/README.md +++ b/README.md @@ -315,6 +315,25 @@ System UI helpers are installed only on supported UI platforms. Shared iOS/visio - on failure it throws `CodeModeToolError` - `call.cancel()` interrupts in-flight JavaScript +### Timers and the event loop + +`setTimeout`/`clearTimeout` are real: callbacks are deferred, delays are honoured, +`clearTimeout` cancels, and an error thrown inside a callback becomes a +`TIMER_CALLBACK_ERROR` diagnostic rather than propagating to whoever called +`setTimeout`. So `await new Promise(r => setTimeout(r, 1000))` — the standard +backoff — actually waits instead of spinning. A long wait remains cancellable and +is still bounded by `timeoutMs`. + +There is no I/O event loop yet: the bridge ABI is synchronous, so native calls run +one at a time and `Promise.all` over several helpers completes them sequentially. +Because a timer is the only thing that can advance a pending program, a promise +with no resolve path and no queued timer is provably unsettleable and fails +immediately with `JS_RUNTIME_ERROR` instead of running out the clock. + +Bridge calls that fail while the script still returns successfully — typically a +forgotten `await`, which silently discards the rejection — produce a +`BRIDGE_FAILURES_NOT_SURFACED` warning diagnostic. + ### Timeout and cancellation `timeoutMs` and `cancel()` are enforced preemptively: a JavaScriptCore execution diff --git a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift index 50fb554..2049743 100644 --- a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift +++ b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift @@ -128,7 +128,9 @@ public enum CodeModeAgentToolDescriptions { Calling conventions: apple.* and other namespaced helpers take one object argument, for example apple.fs.read({ path: "tmp:file.txt" }) and often return structured objects such as { text, base64 }. Node-style filesystem aliases under fs.promises.* use positional arguments, for example fs.promises.readFile("tmp:file.txt", "utf8"), and return Node-like values such as a string for readFile. When in doubt, use the ref.dts from searchJavaScriptAPI — it states the exact signature. fetch(url, options) is a global helper and returns a Response-like object with text(), json(), headers.get(), status, and ok. - Return semantics: the runtime wraps your code in an async function. For multi-statement code, return the final graded value with an explicit top-level return statement. A script that is only a bare final top-level await expression, such as await apple.fs.read({ path: "tmp:file.txt" }), returns that awaited value. Do not use an unreturned async IIFE as the final expression. setTimeout callbacks fire synchronously in this runtime. + Return semantics: the runtime wraps your code in an async function. For multi-statement code, return the final graded value with an explicit top-level return statement. A script that is only a bare final top-level await expression, such as await apple.fs.read({ path: "tmp:file.txt" }), returns that awaited value. Do not use an unreturned async IIFE as the final expression. + + Concurrency: setTimeout and clearTimeout work normally — callbacks are deferred and delays are honoured, so `await new Promise(r => setTimeout(r, 1000))` really waits. There is no I/O event loop, though: native calls run one at a time, so Promise.all over several helpers completes them sequentially rather than concurrently. A promise with no resolve path and no pending timer can never settle and fails fast with JS_RUNTIME_ERROR rather than running out the clock. Allowlisting: include only the required built-in capabilities in allowedCapabilities and only custom provider keys in allowedCapabilityKeys. The two fields are not interchangeable: a built-in capability listed in allowedCapabilityKeys is ignored. These fields declare what your script needs; the host app applies its own ceiling on top, so a capability you list may still be withheld. Execution defaults to a 10000ms timeout and returns structured CodeModeToolError failures for syntax errors, missing JS helpers, runtime throws, validation failures, permission denials, timeouts, cancellation, and internal errors. @@ -139,6 +141,12 @@ public enum CodeModeAgentToolDescriptions { UI_PRESENTER_UNAVAILABLE: host configuration issue; do not retry the same call. INVALID_ARGUMENTS: use the catalog requiredArguments, optionalArguments, argumentHints, and example. NETWORK_POLICY_VIOLATION: the host's network access policy refused the destination; do not retry the same URL and do not add capabilities to work around it. + JS_RUNTIME_ERROR "can never settle": a promise in your script has no resolve path; await every promise and make sure each one resolves. + + Warning diagnostics worth acting on: + BRIDGE_FAILURES_NOT_SURFACED: a helper failed but your script still returned successfully — usually a missing await, which discards the rejection. + RESULT_TRUNCATED: the return value exceeded the size limit; return a summary or write the payload to the sandbox and return the path. + TIMER_CALLBACK_ERROR: a setTimeout callback threw; its error cannot reach the caller, so handle it inside the callback. """ ) diff --git a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift index b805d09..2d3ff06 100644 --- a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift +++ b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift @@ -21,6 +21,7 @@ public final class BridgeInvocationContext: @unchecked Sendable { private let lock = NSLock() private var validatedPermissions: Set = [] + private var failedCapabilityInvocations: [(capability: String, code: String)] = [] private let transcript: ExecutionTranscript private let cancellationController: ExecutionCancellationController @@ -118,6 +119,23 @@ public final class BridgeInvocationContext: @unchecked Sendable { transcript.record(diagnostic: diagnostic) } + /// Notes a capability call that failed, so a script that finishes + /// *successfully* despite failed bridge calls can be flagged. Nothing installs + /// an unhandled-rejection hook — a forgotten `await` is the most common LLM + /// JavaScript mistake, and it silently discards a `CAPABILITY_DENIED` while + /// the agent is told the run succeeded. + func recordCapabilityFailure(capability: String, code: String) { + lock.lock() + failedCapabilityInvocations.append((capability, code)) + lock.unlock() + } + + func capabilityFailures() -> [(capability: String, code: String)] { + lock.lock() + defer { lock.unlock() } + return failedCapabilityInvocations + } + func checkCancellation() throws { // Bridge handlers run on a GCD worker, not inside a Task, so // `Task.isCancelled` was always false here. The controller is the signal diff --git a/Sources/CodeMode/Runtime/BridgeRuntime.swift b/Sources/CodeMode/Runtime/BridgeRuntime.swift index 95f7873..3742e99 100644 --- a/Sources/CodeMode/Runtime/BridgeRuntime.swift +++ b/Sources/CodeMode/Runtime/BridgeRuntime.swift @@ -295,6 +295,7 @@ final class BridgeRuntime: @unchecked Sendable { capabilityKey: capabilityKey, invocationContext: invocationContext ) + invocationContext.recordCapabilityFailure(capability: capability, code: errorPayload.code) invocationContext.log(.error, message: "Capability failed \(capability): \(errorPayload.message)") invocationContext.auditLogger.log(AuditEvent(capability: capability, message: "failed: \(errorPayload.message)")) @@ -521,6 +522,7 @@ final class BridgeRuntime: @unchecked Sendable { switch settlement { case .fulfilled: + recordSwallowedFailures(invocationContext: invocationContext) let output = try decodeOutput( from: context, watchdog: watchdog, @@ -541,12 +543,20 @@ final class BridgeRuntime: @unchecked Sendable { case rejected } - /// Polls the wrapped promise's settled state until it resolves, is - /// cancelled, or the watchdog deadline passes. The settled state is checked - /// before the deadline on every iteration, so a script that has already - /// fulfilled — the common case under the synchronous bridge model — returns - /// its result even if evaluation finished a hair past the deadline, instead - /// of being discarded with a spurious timeout. + /// Runs the runtime's event loop until the wrapped promise settles, the + /// deadline passes, or the program is provably stuck. + /// + /// The settled state is checked before the deadline on every iteration, so a + /// script that has already fulfilled returns its result even if evaluation + /// finished a hair past the deadline, instead of being discarded with a + /// spurious timeout. + /// + /// When the promise is still pending, the *only* thing that can advance it + /// under the synchronous bridge model is a queued timer — microtasks have + /// already drained inside `evaluateScript`, and no native call is + /// outstanding. So this either sleeps until the next timer is due and fires + /// it, or, when no timer is queued, reports immediately that the promise can + /// never settle instead of sleeping a thread to the deadline for nothing. private func waitForSettlement( context: JSContext, watchdog: ExecutionWatchdog, @@ -584,9 +594,126 @@ final class BridgeRuntime: @unchecked Sendable { if watchdog.hasPassedDeadline { throw toolError(code: timeoutCode, message: timeoutMessage, transcript: invocationContext) } - Thread.sleep(forTimeInterval: 0.01) + + let nextTimerDelay = context + .evaluateScript("globalThis.__codemode.nextTimerDelay()")? + .toNumber()? + .doubleValue ?? -1 + + guard nextTimerDelay >= 0 else { + throw toolError( + code: "JS_RUNTIME_ERROR", + message: "Script finished with a promise that can never settle: nothing is left to resolve it. This runtime has no I/O event loop, so only a pending setTimeout can advance a program after the synchronous work is done.", + transcript: invocationContext, + suggestions: [ + "Await every promise the script creates, and make sure each one has a resolve path.", + "A `new Promise(...)` whose executor never calls resolve or reject will hang forever here.", + ] + ) + } + + try runDueTimers( + delayMs: nextTimerDelay, + context: context, + watchdog: watchdog, + cancellationController: cancellationController, + invocationContext: invocationContext, + timeoutCode: timeoutCode, + timeoutMessage: timeoutMessage, + cancelMessage: cancelMessage + ) + } + } + } + + /// Sleeps out a timer's delay — bounded by the deadline and responsive to + /// cancellation — then fires everything that has come due. + private func runDueTimers( + delayMs: Double, + context: JSContext, + watchdog: ExecutionWatchdog, + cancellationController: ExecutionCancellationController, + invocationContext: BridgeInvocationContext, + timeoutCode: String, + timeoutMessage: String, + cancelMessage: String + ) throws { + let start = ContinuousClock.now + let target = start.advanced(by: .milliseconds(Int(delayMs.rounded(.up)))) + + while ContinuousClock.now < target { + if cancellationController.isCancelled { + throw toolError(code: "CANCELLED", message: cancelMessage, transcript: invocationContext) + } + if watchdog.hasPassedDeadline { + throw toolError(code: timeoutCode, message: timeoutMessage, transcript: invocationContext) } + // Short slices so cancellation and the deadline stay responsive + // during a long backoff. + Thread.sleep(forTimeInterval: 0.005) } + + let elapsed = Double((ContinuousClock.now - start).components.attoseconds) / 1e15 + + Double((ContinuousClock.now - start).components.seconds) * 1_000 + _ = context.evaluateScript("globalThis.__codemode.advanceTimers(\(max(elapsed, delayMs)))") + + recordTimerErrors(from: context, invocationContext: invocationContext) + } + + /// An error thrown by a timer callback cannot propagate to whoever called + /// `setTimeout`, so it surfaces as a diagnostic the way an uncaught task + /// error does in a real event loop. + private func recordTimerErrors(from context: JSContext, invocationContext: BridgeInvocationContext) { + guard let json = context.evaluateScript("JSON.stringify(globalThis.__codemode.takeTimerErrors())")?.toString(), + let data = json.data(using: .utf8), + let messages = try? JSONDecoder.codeModeBridge.decode([String].self, from: data), + messages.isEmpty == false + else { + return + } + + for message in messages { + invocationContext.recordDiagnostic( + ToolDiagnostic( + severity: .warning, + code: "TIMER_CALLBACK_ERROR", + message: "A setTimeout callback threw and the error was discarded: \(message)", + category: "execution", + suggestions: ["Handle errors inside timer callbacks; they cannot propagate to the caller of setTimeout."] + ) + ) + } + } + + /// Flags bridge calls that failed while the script still reported success. + /// + /// The runtime installs no unhandled-rejection hook, so a forgotten `await` — + /// the most common LLM JavaScript mistake — discards a `CAPABILITY_DENIED` + /// and the agent is told the run completed. This does not prove the failure + /// was unhandled (a deliberate try/catch looks the same), so it is a warning + /// rather than an error, but it makes the silent case visible. + private func recordSwallowedFailures(invocationContext: BridgeInvocationContext) { + let failures = invocationContext.capabilityFailures() + guard failures.isEmpty == false else { + return + } + + let summary = failures + .map { "\($0.capability) (\($0.code))" } + .joined(separator: ", ") + + invocationContext.recordDiagnostic( + ToolDiagnostic( + severity: .warning, + code: "BRIDGE_FAILURES_NOT_SURFACED", + message: "The script completed successfully, but \(failures.count) bridge call(s) failed and the failure did not reach the result: \(summary).", + category: "execution", + suggestions: [ + "If that was not deliberate, check for a missing await — an un-awaited helper's rejection is silently discarded here.", + "If it was deliberate, ignore this; the result is what the script returned.", + ] + ) + ) } private static func noReturnValueDiagnostic(for code: String) -> ToolDiagnostic { diff --git a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift index 45d2855..f4d288c 100644 --- a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift +++ b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift @@ -1,6 +1,79 @@ import Foundation enum RuntimeJavaScript { + /// Shared by the execution and search runtimes. + private static let timerRuntime = """ + // A real timer queue, drained by the host between JavaScript turns. + // + // `setTimeout` used to invoke its callback synchronously and ignore the + // delay, which made three things wrong at once: `clearTimeout` could never + // cancel, an error thrown by the callback propagated to setTimeout's *caller* + // rather than being an uncaught task error, and + // `await new Promise(r => setTimeout(r, 2000))` — the standard backoff — was + // a hot loop that hammered remote APIs through `fetch`. + globalThis.__codemode.timers = { nextID: 1, entries: [], errors: [] }; + + globalThis.setTimeout = function(fn, delay) { + if (typeof fn !== 'function') { + return 0; + } + const timers = globalThis.__codemode.timers; + const id = timers.nextID++; + timers.entries.push({ + id: id, + fn: fn, + args: Array.prototype.slice.call(arguments, 2), + dueIn: Math.max(0, Number(delay) || 0) + }); + return id; + }; + + globalThis.clearTimeout = function(id) { + const timers = globalThis.__codemode.timers; + timers.entries = timers.entries.filter(function(entry){ return entry.id !== id; }); + }; + + // Milliseconds until the earliest pending timer, or -1 when none are queued. + // -1 means nothing can advance the program: under this runtime's synchronous + // bridge there is no other source of asynchrony. + globalThis.__codemode.nextTimerDelay = function() { + const entries = globalThis.__codemode.timers.entries; + if (entries.length === 0) { + return -1; + } + return entries.reduce(function(min, entry){ return Math.min(min, entry.dueIn); }, Infinity); + }; + + globalThis.__codemode.advanceTimers = function(elapsedMs) { + const timers = globalThis.__codemode.timers; + const due = []; + const remaining = []; + timers.entries.forEach(function(entry){ + entry.dueIn -= elapsedMs; + if (entry.dueIn <= 0) { due.push(entry); } else { remaining.push(entry); } + }); + timers.entries = remaining; + // Registration order breaks ties, matching a real event loop. + due.sort(function(a, b){ return a.id - b.id; }); + due.forEach(function(entry){ + try { + entry.fn.apply(null, entry.args); + } catch (error) { + // An uncaught error in a task cannot propagate to whoever called + // setTimeout; the host reports it as a diagnostic instead. + timers.errors.push(String(error)); + } + }); + return due.length; + }; + + globalThis.__codemode.takeTimerErrors = function() { + const errors = globalThis.__codemode.timers.errors; + globalThis.__codemode.timers.errors = []; + return errors; + }; + """ + static func pruningScript(removingJavaScriptNames names: some Sequence) -> String { let bindingsToRemove = Array(Set(names)).sorted() @@ -123,13 +196,7 @@ enum RuntimeJavaScript { error: function(){ __searchConsole('error', Array.from(arguments).map(function(v){ return String(v); }).join(' ')); } }; - globalThis.setTimeout = function(fn, delay) { - if (typeof fn === 'function') { - fn(); - } - return 0; - }; - globalThis.clearTimeout = function(_) {}; + \(timerRuntime) """ static let bootstrap = """ @@ -200,13 +267,7 @@ enum RuntimeJavaScript { error: function(){ __nativeConsoleLog(Array.from(arguments).map(function(v){ return String(v); }).join(' ')); } }; - globalThis.setTimeout = function(fn, delay) { - if (typeof fn === 'function') { - fn(); - } - return 0; - }; - globalThis.clearTimeout = function(_) {}; + \(timerRuntime) if (typeof URLSearchParams === 'undefined') { globalThis.URLSearchParams = function(initial){ diff --git a/Sources/CodeModeEvaluation/EvalScenarios.swift b/Sources/CodeModeEvaluation/EvalScenarios.swift index 455b624..da0a636 100644 --- a/Sources/CodeModeEvaluation/EvalScenarios.swift +++ b/Sources/CodeModeEvaluation/EvalScenarios.swift @@ -16,6 +16,8 @@ public enum CodeModeEvalScenarios { filesystemCapabilityDenied, executionConsoleLogs, executionTimeout, + executionUnsettleablePromise, + executionTimerBackoff, reminderCatalogDiscovery, catalogFileSystemReadShape, catalogConsoleDiagnostics, @@ -491,10 +493,10 @@ public enum CodeModeEvalScenarios { ) ) - public static let executionTimeout = CodeModeEvalScenario( - id: "execution.timeout", - title: "Unresolved promises time out", - task: "Await a never-resolving JavaScript promise. Do not catch the error in JavaScript; let executeJavaScript surface the structured execution-timeout error.", + public static let executionUnsettleablePromise = CodeModeEvalScenario( + id: "execution.unsettleable-promise", + title: "Unresolvable promises are reported, not waited out", + task: "Await a never-resolving JavaScript promise. Do not catch the error in JavaScript; let executeJavaScript surface the structured error.", executeCode: """ await new Promise(() => {}); return { never: true }; @@ -505,10 +507,49 @@ public enum CodeModeEvalScenarios { toolOrder: [.executeJavaScript], exactAllowedCapabilities: [], requiredExecuteCodeFragments: ["new Promise"], + expectedErrorCode: "JS_RUNTIME_ERROR" + ) + ) + + public static let executionTimeout = CodeModeEvalScenario( + id: "execution.timeout", + title: "CPU-bound scripts hit the execution timeout", + task: "Run a CPU-bound infinite loop. Do not catch the error in JavaScript; let executeJavaScript surface the structured execution-timeout error.", + executeCode: """ + while (true) {} + """, + allowedCapabilities: [], + timeoutMs: 200, + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript], + exactAllowedCapabilities: [], + requiredExecuteCodeFragments: ["while"], expectedErrorCode: "EXECUTION_TIMEOUT" ) ) + public static let executionTimerBackoff = CodeModeEvalScenario( + id: "execution.timer-backoff", + title: "setTimeout honours its delay", + task: "Use setTimeout to wait before returning, and cancel a second timer with clearTimeout so its callback never runs.", + executeCode: """ + const marks = []; + const cancelled = setTimeout(() => { marks.push('cancelled'); }, 5); + clearTimeout(cancelled); + await new Promise(resolve => setTimeout(resolve, 20)); + marks.push('resumed'); + return { marks }; + """, + allowedCapabilities: [], + timeoutMs: 2_000, + expectation: CodeModeEvalExpectation( + toolOrder: [.executeJavaScript], + exactAllowedCapabilities: [], + requiredExecuteCodeFragments: ["setTimeout"], + expectedOutput: .object(["marks": .array([.string("resumed")])]) + ) + ) + public static let reminderCatalogDiscovery = CodeModeEvalScenario( id: "catalog.reminder-create", title: "Reminder helper discovery", diff --git a/Tests/CodeModeTests/EventLoopTests.swift b/Tests/CodeModeTests/EventLoopTests.swift new file mode 100644 index 0000000..5db3b4e --- /dev/null +++ b/Tests/CodeModeTests/EventLoopTests.swift @@ -0,0 +1,224 @@ +import Foundation +import Testing +@testable import CodeMode + +// `setTimeout` used to invoke its callback synchronously and ignore the delay. +// Three things were wrong at once: clearTimeout could never cancel, an error +// thrown by a callback propagated to setTimeout's *caller*, and +// `await new Promise(r => setTimeout(r, 2000))` — the standard backoff — was a +// hot loop that hammered remote APIs through fetch. + +@Test func setTimeoutDefersItsCallbackInsteadOfRunningItInline() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => { order.push('timer'); }, 0); + order.push('sync'); + await new Promise(resolve => setTimeout(resolve, 1)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.error == nil) + let order = try #require(observed.result?.output?.objectValue?.array("order")) + #expect(order == [.string("sync"), .string("timer")]) +} + +@Test func setTimeoutActuallyWaitsTheRequestedDelay() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 150)); return 'done';", + allowedCapabilities: [], + timeoutMs: 5_000 + ) + ) + let elapsed = Date().timeIntervalSince(started) + + #expect(observed.result?.output == .string("done")) + // The delay was previously ignored entirely, so a backoff loop spun. + #expect(elapsed >= 0.14) + #expect(elapsed < 3) +} + +@Test func clearTimeoutCancelsAPendingCallback() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const fired = []; + const id = setTimeout(() => { fired.push('should-not-run'); }, 5); + clearTimeout(id); + await new Promise(resolve => setTimeout(resolve, 20)); + return { fired }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output?.objectValue?.array("fired") == []) +} + +@Test func timersFireInRegistrationOrderWhenDueTogether() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => order.push('b'), 10); + setTimeout(() => order.push('a'), 5); + await new Promise(resolve => setTimeout(resolve, 30)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output?.objectValue?.array("order") == [.string("a"), .string("b")]) +} + +@Test func anErrorInATimerCallbackBecomesADiagnosticNotACallerThrow() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + setTimeout(() => { throw new Error('boom'); }, 1); + await new Promise(resolve => setTimeout(resolve, 20)); + return 'survived'; + """, + allowedCapabilities: [] + ) + ) + + // Previously this propagated to whoever called setTimeout, which is not how + // an event loop behaves. + #expect(observed.result?.output == .string("survived")) + let diagnostics = try #require(observed.result?.diagnostics) + #expect(diagnostics.contains { $0.code == "TIMER_CALLBACK_ERROR" && $0.message.contains("boom") }) +} + +@Test func anUnsettleablePromiseIsReportedImmediately() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // No resolve path and no queued timer: nothing in this runtime can advance + // it, so sleeping a thread to the deadline buys nothing. + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(() => {}); return 1;", + allowedCapabilities: [], + timeoutMs: 30_000 + ) + ) + + #expect(observed.error?.code == "JS_RUNTIME_ERROR") + #expect(observed.error?.message.contains("can never settle") == true) + #expect(Date().timeIntervalSince(started) < 5) +} + +@Test func aTimerStillPendingAtTheDeadlineTimesOut() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // A timer due long after the deadline must not be waited out. + let started = Date() + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 30000)); return 1;", + allowedCapabilities: [], + timeoutMs: 100 + ) + ) + + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + #expect(Date().timeIntervalSince(started) < 5) +} + +@Test func aLongBackoffStaysCancellable() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 20000)); return 1;", + allowedCapabilities: [], + timeoutMs: 60_000 + ) + ) + + Task { + try? await Task.sleep(nanoseconds: 100_000_000) + call.cancel() + } + + let started = Date() + let observed = await observe(call) + #expect(observed.error?.code == "CANCELLED") + #expect(Date().timeIntervalSince(started) < 10) +} + +// MARK: - Swallowed bridge failures + +@Test func aForgottenAwaitOnAFailedCallIsSurfacedAsADiagnostic() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Nothing installs an unhandled-rejection hook, so this used to complete with + // the agent told "done" while a CAPABILITY_DENIED was silently discarded. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + apple.keychain.get({ key: 'never-allowed' }).catch(() => {}); + await new Promise(resolve => setTimeout(resolve, 10)); + return 'looks fine'; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output == .string("looks fine")) + let diagnostics = try #require(observed.result?.diagnostics) + let swallowed = try #require(diagnostics.first { $0.code == "BRIDGE_FAILURES_NOT_SURFACED" }) + #expect(swallowed.message.contains("keychain.read")) + #expect(swallowed.message.contains("CAPABILITY_DENIED")) + #expect(swallowed.suggestions.contains { $0.contains("missing await") }) +} + +@Test func aCleanRunGetsNoSwallowedFailureDiagnostic() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await apple.fs.write({ path: 'tmp:ok.txt', data: 'x' }); return 'ok';", + allowedCapabilities: [.fsWrite] + ) + ) + + #expect(observed.result?.diagnostics.contains { $0.code == "BRIDGE_FAILURES_NOT_SURFACED" } == false) +} diff --git a/Tests/CodeModeTests/HostRuntimeTests.swift b/Tests/CodeModeTests/HostRuntimeTests.swift index 158798a..b44c8b4 100644 --- a/Tests/CodeModeTests/HostRuntimeTests.swift +++ b/Tests/CodeModeTests/HostRuntimeTests.swift @@ -602,7 +602,11 @@ import Testing ) #expect(observed.result == nil) - #expect(observed.error?.code == "EXECUTION_TIMEOUT") + // A promise with no resolve path and no pending timer is provably + // unsettleable under this runtime, so it is reported immediately instead of + // sleeping a thread until the deadline for a result that can never arrive. + #expect(observed.error?.code == "JS_RUNTIME_ERROR") + #expect(observed.error?.message.contains("can never settle") == true) } @Test func executeRecoversWithFreshContextAfterTimeout() async throws { @@ -617,7 +621,7 @@ import Testing timeoutMs: 30 ) ) - #expect(timedOut.error?.code == "EXECUTION_TIMEOUT") + #expect(timedOut.error?.code == "JS_RUNTIME_ERROR") let recovered = try await execute( tools, @@ -700,13 +704,17 @@ import Testing let (tools, sandbox) = try makeTools() defer { cleanup(sandbox) } + // A long timer, not an unsettleable promise: the latter now fails fast with + // JS_RUNTIME_ERROR, which would win the race against the cancellation this + // test is about. let call = try await tools.executeJavaScript( JavaScriptExecutionRequest( code: """ - await new Promise(() => {}); + await new Promise(resolve => setTimeout(resolve, 20000)); return { never: true }; """, - allowedCapabilities: [] + allowedCapabilities: [], + timeoutMs: 60_000 ) ) From 7f44ebbe9621c68ed44ef285edeb476ae615ac1a Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 20:43:34 -0700 Subject: [PATCH 12/18] Bound concurrent executions and cover the concurrent path with tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The execution queue is `.concurrent` with no width limit, and each execution blocks its thread for the whole run, so thirty parallel `executeJavaScript` calls meant thirty blocked GCD threads and thirty live JavaScriptCore VMs. There was also no concurrent-execution test at all, despite the concurrent queue being a core design point. `ExecutionLimits.maxConcurrentExecutions` (default 8) gates the queue with a semaphore acquired on the worker rather than before dispatch, so the caller's async context is never blocked and waiting for a slot does not eat the per-execution `timeoutMs`. Two tests: twelve executions through three slots all complete with correct, isolated results and no state carried between contexts; and executions beyond the limit queue rather than fail. Closes the remaining "bound on concurrent executions" item in TODO.md §1. --- Sources/CodeMode/Runtime/BridgeRuntime.swift | 13 +++ .../CodeMode/Runtime/ExecutionLimits.swift | 9 ++- TODO.md | 8 +- Tests/CodeModeTests/EventLoopTests.swift | 81 +++++++++++++++++++ 4 files changed, 107 insertions(+), 4 deletions(-) diff --git a/Sources/CodeMode/Runtime/BridgeRuntime.swift b/Sources/CodeMode/Runtime/BridgeRuntime.swift index 3742e99..8b583b2 100644 --- a/Sources/CodeMode/Runtime/BridgeRuntime.swift +++ b/Sources/CodeMode/Runtime/BridgeRuntime.swift @@ -32,6 +32,14 @@ final class BridgeRuntime: @unchecked Sendable { qos: .userInitiated, attributes: .concurrent ) + /// Caps how many executions hold a thread and a live `JSContext` at once. + /// + /// The queue is concurrent and each execution blocks its thread for the whole + /// run, so thirty parallel `executeJavaScript` calls meant thirty blocked GCD + /// threads and thirty JavaScriptCore VMs. Excess executions now queue instead + /// of exhausting the thread pool; `timeoutMs` still measures the run itself, + /// not the wait for a slot. + private let executionSlots: DispatchSemaphore init( registry: CapabilityRegistry, @@ -43,6 +51,7 @@ final class BridgeRuntime: @unchecked Sendable { self.catalog = catalog self.config = config self.unsupportedBuiltInJavaScriptNames = unsupportedBuiltInJavaScriptNames + self.executionSlots = DispatchSemaphore(value: max(1, config.executionLimits.maxConcurrentExecutions)) } func search(_ request: JavaScriptAPISearchRequest) throws -> JavaScriptAPISearchResponse { @@ -179,6 +188,10 @@ final class BridgeRuntime: @unchecked Sendable { ) async throws -> Output { try await withCheckedThrowingContinuation { continuation in executionQueue.async { + // Wait for a slot on the worker, not before dispatching, so the + // caller's async context is never blocked. + self.executionSlots.wait() + defer { self.executionSlots.signal() } do { continuation.resume(returning: try operation()) } catch { diff --git a/Sources/CodeMode/Runtime/ExecutionLimits.swift b/Sources/CodeMode/Runtime/ExecutionLimits.swift index 3a59c7e..bcbb891 100644 --- a/Sources/CodeMode/Runtime/ExecutionLimits.swift +++ b/Sources/CodeMode/Runtime/ExecutionLimits.swift @@ -33,6 +33,11 @@ public struct ExecutionLimits: Sendable, Equatable { /// Maximum retained permission events. public var maxPermissionEvents: Int + /// How many executions may hold a thread and a live JavaScriptCore context + /// at once. Excess executions queue rather than exhausting the GCD thread + /// pool; the per-execution `timeoutMs` measures the run, not the wait. + public var maxConcurrentExecutions: Int + /// Depth of the events `AsyncStream` buffer. A host that stops draining /// drops the oldest events rather than growing without bound. public var maxBufferedEvents: Int @@ -44,7 +49,8 @@ public struct ExecutionLimits: Sendable, Equatable { maxLogMessageCharacters: Int = 8_000, maxDiagnostics: Int = 200, maxPermissionEvents: Int = 200, - maxBufferedEvents: Int = 1_000 + maxBufferedEvents: Int = 1_000, + maxConcurrentExecutions: Int = 8 ) { self.maxResultCharacters = maxResultCharacters self.truncatedResultPreviewCharacters = truncatedResultPreviewCharacters @@ -53,6 +59,7 @@ public struct ExecutionLimits: Sendable, Equatable { self.maxDiagnostics = maxDiagnostics self.maxPermissionEvents = maxPermissionEvents self.maxBufferedEvents = maxBufferedEvents + self.maxConcurrentExecutions = maxConcurrentExecutions } public static let standard = ExecutionLimits() diff --git a/TODO.md b/TODO.md index 0392115..c7516b2 100644 --- a/TODO.md +++ b/TODO.md @@ -37,9 +37,11 @@ just the watchdog: serialization, cancellation, catch-proof termination, context recovery). - [ ] **JS heap / memory cap** — nothing bounds `JSContext`/`JSContextGroup` heap; `new Array(1e9)` can still exhaust host memory. NOT addressed. -- [ ] **Bound on concurrent executions** — `executionQueue` is `.concurrent` - with spin-waiting workers (`BridgeRuntime.swift:25-29`), so N long-running - scripts occupy N threads. No concurrency cap. NOT addressed. +- [x] **Bound on concurrent executions** — `ExecutionLimits.maxConcurrentExecutions` + (default 8) gates `runOnExecutionQueue` with a semaphore acquired *on* the + worker, so excess executions queue instead of exhausting the GCD thread pool. + Covered by `concurrentExecutionsAllCompleteAndStayIsolated` and + `executionsBeyondTheSlotLimitQueueRatherThanFail`. - [ ] `runOnExecutionQueue` still has no task-cancellation handler wired into the dispatched block (best-effort only). NOT addressed. diff --git a/Tests/CodeModeTests/EventLoopTests.swift b/Tests/CodeModeTests/EventLoopTests.swift index 5db3b4e..6561b86 100644 --- a/Tests/CodeModeTests/EventLoopTests.swift +++ b/Tests/CodeModeTests/EventLoopTests.swift @@ -222,3 +222,84 @@ import Testing #expect(observed.result?.diagnostics.contains { $0.code == "BRIDGE_FAILURES_NOT_SURFACED" } == false) } + +// MARK: - Bounded concurrency + +@Test func concurrentExecutionsAllCompleteAndStayIsolated() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 3) + ) + defer { cleanup(sandbox) } + + // The concurrent execution queue is a core design point but had no test, and + // no width limit: each execution blocks its thread for the whole run, so N + // parallel calls meant N blocked GCD threads and N live JavaScriptCore VMs. + // More executions than slots must still all complete, each with its own + // context and its own result. + let results = try await withThrowingTaskGroup(of: (Int, JSONValue?).self) { group in + for index in 0..<12 { + group.addTask { + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.fs.write({ path: 'tmp:worker-\(index).txt', data: 'value-\(index)' }); + const read = await apple.fs.read({ path: 'tmp:worker-\(index).txt' }); + return { index: \(index), text: read.text, leaked: typeof globalThis.__leaked }; + """, + allowedCapabilities: [.fsRead, .fsWrite], + timeoutMs: 20_000 + ) + ) + return (index, observed.result?.output) + } + } + + var collected: [Int: JSONValue?] = [:] + for try await (index, output) in group { + collected[index] = output + } + return collected + } + + #expect(results.count == 12) + for index in 0..<12 { + let output = try #require(results[index]??.objectValue) + #expect(output.int("index") == index) + #expect(output.string("text") == "value-\(index)") + // A fresh JSContext per execution: no state carries between them. + #expect(output.string("leaked") == "undefined") + } +} + +@Test func executionsBeyondTheSlotLimitQueueRatherThanFail() async throws { + let (tools, sandbox) = try makeTools( + executionLimits: ExecutionLimits(maxConcurrentExecutions: 1) + ) + defer { cleanup(sandbox) } + + // With one slot these serialize; the point is that the extras wait for a slot + // instead of erroring, and that waiting for a slot does not count against the + // per-execution timeout. + let outputs = try await withThrowingTaskGroup(of: JSONValue?.self) { group in + for index in 0..<4 { + group.addTask { + try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(r => setTimeout(r, 60)); return \(index);", + allowedCapabilities: [], + timeoutMs: 500 + ) + ).result?.output + } + } + var collected: [JSONValue?] = [] + for try await output in group { + collected.append(output) + } + return collected + } + + #expect(outputs.compactMap { $0?.intValue }.sorted() == [0, 1, 2, 3]) +} From 7245335e8e07f8d35f6284c9da5be0bc5d1d9ec8 Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 20:49:50 -0700 Subject: [PATCH 13/18] Unfreeze the catalog and bring host providers toward built-in parity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Partial P1 #13. The provider-per-method macro and the MCP adapter remain; this removes the blocker they both sit behind and closes the metadata gaps. **The catalog was frozen at init.** `BridgeCatalog` snapshotted the registry in `CodeModeAgentTools.init`, so `CapabilityRegistry`'s public post-init `register(...)` methods were unreachable in the supported flow — and if reached, would have desynced what search advertises from what execution can invoke. The registry now carries a generation counter and the catalog rebuilds when it changes, so the two can no longer disagree. `CodeModeAgentTools.register(provider:)` exposes this: a host whose domain APIs depend on runtime state — a signed-in account, a loaded document — can register them when that state arrives, and search sees them immediately. **Host providers can express constrained values.** `@CodeMode` rejected any non-primitive property type, so a host's `allowedStringValues` was always empty while built-ins advertised theirs — the model had no way to learn a provider's accepted spellings. An unrecognized simple type is now taken to be a `CodeModeStringEnum`, exactly as `@BuiltInCodeMode` already assumed, and its accepted values reach the catalog, the generated TypeScript union, and argument validation. A denylist of common non-enum types (`Date`, `URL`, `UUID`, …) keeps the "unsupported property type" diagnostic pointing at the property rather than letting a natural mistake fail inside generated code. **Generated metadata is no longer vacuous.** `example` was always `await path({})` regardless of arguments; it is now a call the model can pattern-match, with a placeholder per required argument. `tags` was only the parent path, so `myapp.tasks.complete` was findable only by searching "myapp.tasks"; every path segment is now a tag. --- Sources/CodeMode/Catalog/BridgeCatalog.swift | 77 +++++++++++----- .../CodeMode/Host/CodeModeAgentTools.swift | 17 ++++ .../Registry/CapabilityRegistry.swift | 16 ++++ Sources/CodeModeAuthoring/README.md | 23 ++++- Sources/CodeModeMacros/CodeModeMacro.swift | 90 ++++++++++++++++++- .../CodeModeMacroExpansionTests.swift | 83 ++++++++++++++++- .../CodeModeTests/CodeModeProviderTests.swift | 75 ++++++++++++++++ 7 files changed, 352 insertions(+), 29 deletions(-) diff --git a/Sources/CodeMode/Catalog/BridgeCatalog.swift b/Sources/CodeMode/Catalog/BridgeCatalog.swift index 6e65647..3583424 100644 --- a/Sources/CodeMode/Catalog/BridgeCatalog.swift +++ b/Sources/CodeMode/Catalog/BridgeCatalog.swift @@ -1,25 +1,54 @@ import Foundation -struct BridgeCatalog: Sendable { +/// The model-facing view of the registry. +/// +/// Tracks the registry rather than snapshotting it once at init. The snapshot +/// used to be frozen in `CodeModeAgentTools.init`, which made +/// `CapabilityRegistry`'s public post-init `register(...)` methods either +/// unreachable in the supported flow or — if reached — a way to desync what +/// search advertises from what execution can actually invoke. +final class BridgeCatalog: @unchecked Sendable { private struct SearchCatalogPayload: Sendable, Codable { var references: [JavaScriptAPIReference] var byCapability: [String: JavaScriptAPIReference] var byJSName: [String: JavaScriptAPIReference] } - private let references: [JavaScriptAPIReference] - private let referencesByCapability: [CodeModeCapabilityKey: JavaScriptAPIReference] - private let searchCatalog: JSONValue - private let allJavaScriptNames: [String] + private struct Snapshot { + var generation: Int + var references: [JavaScriptAPIReference] + var referencesByCapability: [CodeModeCapabilityKey: JavaScriptAPIReference] + var searchCatalog: JSONValue + var allJavaScriptNames: [String] + } + + private let registry: CapabilityRegistry + private let lock = NSLock() + private var snapshot: Snapshot init(registry: CapabilityRegistry) { + self.registry = registry + self.snapshot = Self.makeSnapshot(registry: registry) + } + + /// Rebuilds if the registry has changed since the last read. Rebuilding is + /// the whole catalog, but it only happens when a provider is actually + /// registered — not on every search. + private func current() -> Snapshot { + lock.lock() + defer { lock.unlock() } + let generation = registry.generation + if snapshot.generation != generation { + snapshot = Self.makeSnapshot(registry: registry) + } + return snapshot + } + + private static func makeSnapshot(registry: CapabilityRegistry) -> Snapshot { + let generation = registry.generation let functions = registry.allRegisteredFunctions().sorted { $0.catalogCapability < $1.catalogCapability } let references = functions.map(Self.reference(from:)) - self.references = references - self.referencesByCapability = Dictionary(uniqueKeysWithValues: references.map { ($0.capabilityKey, $0) }) - self.allJavaScriptNames = Array(Set(references.flatMap(\.jsNames))).sorted() - var byJSName: [String: JavaScriptAPIReference] = [:] for reference in references { for jsName in reference.jsNames { @@ -27,35 +56,41 @@ struct BridgeCatalog: Sendable { } } - self.searchCatalog = Self.jsonValue( - from: SearchCatalogPayload( - references: references, - byCapability: Dictionary(uniqueKeysWithValues: references.map { ($0.capability, $0) }), - byJSName: byJSName - ) + return Snapshot( + generation: generation, + references: references, + referencesByCapability: Dictionary(references.map { ($0.capabilityKey, $0) }, uniquingKeysWith: { first, _ in first }), + searchCatalog: Self.jsonValue( + from: SearchCatalogPayload( + references: references, + byCapability: Dictionary(references.map { ($0.capability, $0) }, uniquingKeysWith: { first, _ in first }), + byJSName: byJSName + ) + ), + allJavaScriptNames: Array(Set(references.flatMap(\.jsNames))).sorted() ) } func reference(for capability: CapabilityID) -> JavaScriptAPIReference? { - referencesByCapability[capability.codeModeKey] + current().referencesByCapability[capability.codeModeKey] } func reference(for capabilityKey: CodeModeCapabilityKey) -> JavaScriptAPIReference? { - referencesByCapability[capabilityKey] + current().referencesByCapability[capabilityKey] } func allReferences() -> [JavaScriptAPIReference] { - references + current().references } func searchCatalogValue() -> JSONValue { - searchCatalog + current().searchCatalog } /// TypeScript declarations for the whole platform-filtered surface, for a /// host to drop into its system prompt. func typeDeclarations() -> String { - TypeScriptDeclarations.surface(for: references) + TypeScriptDeclarations.surface(for: current().references) } func closestFunctionNames(to candidate: String, limit: Int = 3) -> [String] { @@ -68,7 +103,7 @@ struct BridgeCatalog: Sendable { let distanceThreshold = max(2, min(8, normalizedCandidate.count / 3 + 1)) - let ranked = allJavaScriptNames.map { name in + let ranked = current().allJavaScriptNames.map { name in let normalizedName = name.lowercased() let distance = Self.levenshtein(normalizedCandidate, normalizedName) let prefixBonus = normalizedName.hasPrefix(normalizedCandidate) || normalizedCandidate.hasPrefix(normalizedName) ? -2 : 0 diff --git a/Sources/CodeMode/Host/CodeModeAgentTools.swift b/Sources/CodeMode/Host/CodeModeAgentTools.swift index 5fb7f7f..03e91a8 100644 --- a/Sources/CodeMode/Host/CodeModeAgentTools.swift +++ b/Sources/CodeMode/Host/CodeModeAgentTools.swift @@ -68,4 +68,21 @@ public final class CodeModeAgentTools: @unchecked Sendable { public func capabilities() -> [JavaScriptAPIReference] { catalog.allReferences() } + + /// Adds host providers after construction. + /// + /// Providers no longer have to be known at init: the catalog tracks the + /// registry, so search starts advertising these immediately and the JS + /// bindings are installed for the next execution. A host whose domain APIs + /// depend on runtime state — a signed-in account, a loaded document — can + /// register them when that state arrives. + /// + /// Registering a key that already exists replaces it. + public func register(providers: [any CodeModeProvider]) { + registry.register(providers.flatMap { $0.codeModeRegistrations() }) + } + + public func register(provider: any CodeModeProvider) { + register(providers: [provider]) + } } diff --git a/Sources/CodeMode/Registry/CapabilityRegistry.swift b/Sources/CodeMode/Registry/CapabilityRegistry.swift index a6d16b5..e5bb049 100644 --- a/Sources/CodeMode/Registry/CapabilityRegistry.swift +++ b/Sources/CodeMode/Registry/CapabilityRegistry.swift @@ -435,6 +435,11 @@ public final class CapabilityRegistry: @unchecked Sendable { private let lock = NSLock() private var registrations: [CapabilityID: CapabilityRegistration] = [:] private var codeModeRegistrations: [CodeModeCapabilityKey: CodeModeRegistration] = [:] + /// Bumped on every mutation so `BridgeCatalog` can tell whether its snapshot + /// is still current. Without it the catalog was frozen at init, which made + /// the public `register(...)` methods below either unreachable or a way to + /// desync what search advertises from what execution can actually invoke. + private var generationValue = 0 public init(registrations: [CapabilityRegistration] = [], codeModeRegistrations: [CodeModeRegistration] = []) { for registration in registrations { @@ -445,9 +450,17 @@ public final class CapabilityRegistry: @unchecked Sendable { } } + /// Monotonic version of the registry's contents. + public var generation: Int { + lock.lock() + defer { lock.unlock() } + return generationValue + } + public func register(_ registration: CapabilityRegistration) { lock.lock() registrations[registration.descriptor.id] = registration + generationValue += 1 lock.unlock() } @@ -456,12 +469,14 @@ public final class CapabilityRegistry: @unchecked Sendable { for registration in registrations { self.registrations[registration.descriptor.id] = registration } + generationValue += 1 lock.unlock() } public func register(_ registration: CodeModeRegistration) { lock.lock() codeModeRegistrations[registration.capabilityKey] = registration + generationValue += 1 lock.unlock() } @@ -470,6 +485,7 @@ public final class CapabilityRegistry: @unchecked Sendable { for registration in registrations { self.codeModeRegistrations[registration.capabilityKey] = registration } + generationValue += 1 lock.unlock() } diff --git a/Sources/CodeModeAuthoring/README.md b/Sources/CodeModeAuthoring/README.md index 2dbb997..7a992e9 100644 --- a/Sources/CodeModeAuthoring/README.md +++ b/Sources/CodeModeAuthoring/README.md @@ -77,4 +77,25 @@ the host-owned ceiling that actually enforces access. Macro v1 maps one type to one JavaScript function. `Arguments` must be a nested struct, and no-arg tools use an empty `Arguments` struct. `call(arguments:)` must be `throws` or `async throws`, and it can return `Void` or a nested `Result` struct. -Supported `Arguments` and `Result` property shapes are JSON primitives, `JSONValue`, arrays/dictionaries of JSON-shaped values, and optional forms. Throw `CodeModeFunctionError` for structured failures that should surface as CodeMode bridge errors. +Supported `Arguments` and `Result` property shapes are JSON primitives, `JSONValue`, arrays/dictionaries of JSON-shaped values, optional forms, and `CodeModeStringEnum` types. Throw `CodeModeFunctionError` for structured failures that should surface as CodeMode bridge errors. + +An enum-typed argument advertises its accepted values, so the catalog, the +generated TypeScript declaration, and argument validation all agree: + +```swift +enum Priority: String, CodeModeStringEnum { + case low, normal, high +} + +struct Arguments { + @CodeModeParam("Task identifier") + var id: String + @CodeModeParam("low, normal (default), or high") + var priority: Priority? +} +``` + +Providers do not have to be known at construction. `CodeModeAgentTools.register(provider:)` +adds them later — search advertises them immediately and the JavaScript bindings +are installed for the next execution — which is what a host needs when its domain +APIs depend on runtime state such as a signed-in account. diff --git a/Sources/CodeModeMacros/CodeModeMacro.swift b/Sources/CodeModeMacros/CodeModeMacro.swift index 72f6086..6d19b3f 100644 --- a/Sources/CodeModeMacros/CodeModeMacro.swift +++ b/Sources/CodeModeMacros/CodeModeMacro.swift @@ -116,12 +116,13 @@ public struct CodeModeMacro: ExtensionMacro { jsPath: \(literal(path)), title: \(literal(title(for: path))), summary: \(literal(summary)), - tags: [\(literal(parentPath(for: path)))], - example: "await \(path)({})", + tags: \(tagsSource(for: path)), + example: \(literal(example(for: path, fields: argumentFields.fields))), requiredArguments: [\(argumentFields.fields.filter { !$0.optional }.map { literal($0.name) }.joined(separator: ", "))], optionalArguments: [\(argumentFields.fields.filter(\.optional).map { literal($0.name) }.joined(separator: ", "))], argumentTypes: \(argumentTypesSource(for: argumentFields.fields)), argumentHints: \(argumentHintsSource(for: argumentFields.fields)), + argumentConstraints: \(argumentConstraintsSource(for: argumentFields.fields)), resultSummary: \(literal(returnsVoid ? "null" : resultInfo?.summary ?? "JSON value")), handler: { arguments, _ in try CodeModeAsyncBridge.run { @@ -275,7 +276,8 @@ public struct CodeModeMacro: ExtensionMacro { swiftType: typeInfo.swiftType, argumentType: typeInfo.argumentType, optional: typeInfo.optional, - description: attributeName.flatMap { attributeString(named: $0, attributes: variable.attributes).first } + description: attributeName.flatMap { attributeString(named: $0, attributes: variable.attributes).first }, + isStringEnum: typeInfo.isStringEnum ) ) } @@ -471,6 +473,60 @@ public struct CodeModeMacro: ExtensionMacro { return segments.dropLast().joined(separator: ".") } + /// Every path segment, so a helper is findable by its namespace *and* its + /// domain — the old single parent-path tag made `myapp.tasks.complete` + /// discoverable only by searching for "myapp.tasks". + private static func tagsSource(for path: String) -> String { + let segments = path.split(separator: ".").map(String.init) + var tags: [String] = [] + for segment in segments where tags.contains(segment) == false { + tags.append(segment) + } + if segments.count > 1 { + tags.append(segments.dropLast().joined(separator: ".")) + } + return "[" + tags.map(literal).joined(separator: ", ") + "]" + } + + /// A call the model can pattern-match, rather than the vacuous + /// `await path({})` that was emitted regardless of the arguments. + private static func example(for path: String, fields: [ToolField]) -> String { + let required = fields.filter { $0.optional == false } + guard required.isEmpty == false else { + return "await \(path)({})" + } + let pairs = required.map { field in + "\(field.name): \(placeholder(for: field))" + } + return "await \(path)({ " + pairs.joined(separator: ", ") + " })" + } + + private static func placeholder(for field: ToolField) -> String { + switch field.argumentType { + case "number": + return "0" + case "bool": + return "true" + case "array": + return "[]" + case "object": + return "{}" + default: + return "\"\(field.name)\"" + } + } + + private static func argumentConstraintsSource(for fields: [ToolField]) -> String { + let enumFields = fields.filter(\.isStringEnum) + guard enumFields.isEmpty == false else { + return "CapabilityArgumentConstraints.none" + } + let entries = enumFields.map { field in + "\(literal(field.name)): \(field.swiftType).codeModeAllowedValues" + }.joined(separator: ", ") + return "CapabilityArgumentConstraints(allowedStringValues: [\(entries)])" + } + private static func literal(_ value: String) -> String { let escaped = value .replacingOccurrences(of: "\\", with: "\\\\") @@ -511,6 +567,7 @@ private struct ToolField { var argumentType: String var optional: Bool var description: String? + var isStringEnum = false } private struct TypeInfo { @@ -518,6 +575,7 @@ private struct TypeInfo { var argumentType: String var optional: Bool var isSupported: Bool + var isStringEnum = false init(typeSyntax rawType: String) { let trimmed = rawType.replacingOccurrences(of: " ", with: "") @@ -554,6 +612,15 @@ private struct TypeInfo { if unwrapped.hasPrefix("["), unwrapped.hasSuffix("]") { self.argumentType = unwrapped.contains(":") ? "object" : "array" self.isSupported = Self.isSupportedCollection(unwrapped) + } else if Self.isSimpleIdentifier(unwrapped), Self.knownNonEnumTypes.contains(unwrapped) == false { + // An unrecognized simple type is taken to be a + // `CodeModeStringEnum`, the same assumption @BuiltInCodeMode + // makes. Host providers could not express constrained values at + // all before this, so their `allowedStringValues` was always + // empty while built-ins advertised theirs. + self.argumentType = "string" + self.isStringEnum = true + self.isSupported = true } else { self.argumentType = "any" self.isSupported = false @@ -561,6 +628,23 @@ private struct TypeInfo { } } + /// Types that are plainly not `CodeModeStringEnum`s. Without this list a + /// natural mistake such as `var date: Date` would be taken for an enum and + /// surface as a confusing error inside generated code instead of a macro + /// diagnostic pointing at the property. + private static let knownNonEnumTypes: Set = [ + "Date", "URL", "Data", "UUID", "Decimal", "Any", "AnyObject", "AnyHashable", + "Never", "Void", "Character", "Substring", "Int8", "Int16", "Int32", "Int64", + "UInt", "UInt8", "UInt16", "UInt32", "UInt64", "CGFloat", "NSNumber", "NSString", + ] + + private static func isSimpleIdentifier(_ type: String) -> Bool { + guard let first = type.first, first.isLetter || first == "_" else { + return false + } + return type.allSatisfy { $0.isLetter || $0.isNumber || $0 == "_" || $0 == "." } + } + private static func isSupportedCollection(_ type: String) -> Bool { let scalarTypes: Set = ["String", "Bool", "Int", "Double", "Float", "JSONValue"] let inner = String(type.dropFirst().dropLast()) diff --git a/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift b/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift index a6bed25..513097c 100644 --- a/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift +++ b/Tests/CodeModeAuthoringTests/CodeModeMacroExpansionTests.swift @@ -69,7 +69,7 @@ private struct FailingTaskTool: CodeModeProvider { #expect(registration.jsPath == "myapp.tasks.complete") #expect(registration.title == "complete") #expect(registration.summary == "Mark a task complete.") - #expect(registration.tags == ["myapp.tasks"]) + #expect(registration.tags == ["myapp", "tasks", "complete", "myapp.tasks"]) #expect(registration.requiredArguments == ["id"]) #expect(registration.optionalArguments == ["note"]) #expect(registration.argumentTypes["id"] == .string) @@ -233,12 +233,13 @@ private func observe(_ call: JavaScriptExecutionCall) async -> ObservedExecution jsPath: "macro.api.greet", title: "greet", summary: "Greet a person.", - tags: ["macro.api"], - example: "await macro.api.greet({})", + tags: ["macro", "api", "greet", "macro.api"], + example: "await macro.api.greet({ name: \\"name\\" })", requiredArguments: ["name"], optionalArguments: ["excited"], argumentTypes: ["name": CapabilityArgumentType.string, "excited": CapabilityArgumentType.bool], argumentHints: ["name": "Person name", "excited": "Whether to add emphasis"], + argumentConstraints: CapabilityArgumentConstraints.none, resultSummary: "Greeting payload", handler: { arguments, _ in try CodeModeAsyncBridge.run { @@ -292,12 +293,13 @@ private func observe(_ call: JavaScriptExecutionCall) async -> ObservedExecution jsPath: "macro.api.ping", title: "ping", summary: "Ping.", - tags: ["macro.api"], + tags: ["macro", "api", "ping", "macro.api"], example: "await macro.api.ping({})", requiredArguments: [], optionalArguments: [], argumentTypes: [:], argumentHints: [:], + argumentConstraints: CapabilityArgumentConstraints.none, resultSummary: "null", handler: { arguments, _ in try CodeModeAsyncBridge.run { @@ -511,3 +513,76 @@ private func observe(_ call: JavaScriptExecutionCall) async -> ObservedExecution macros: codeModeTestMacros() ) } + +// MARK: - Constrained argument values for host providers + +/// Host providers could not express constrained values at all: the macro +/// rejected any non-primitive property type, so their `allowedStringValues` was +/// always empty while built-ins advertised theirs. This is the parity fix. +enum DemoPriority: String, CodeModeStringEnum { + case low + case normal + case high +} + +@CodeMode(path: "macro.api.prioritize", description: "Set a priority.") +struct PrioritizeTool: Sendable { + struct Arguments { + @CodeModeParam("Task identifier") + var id: String + @CodeModeParam("low, normal (default), or high") + var priority: DemoPriority? + } + + struct Result { + var id: String + var priority: String + } + + func call(arguments: Arguments) throws -> Result { + Result(id: arguments.id, priority: (arguments.priority ?? .normal).rawValue) + } +} + +@Test func codeModeMacroAdvertisesEnumConstraintsForHostProviders() throws { + let registration = try #require(PrioritizeTool().codeModeRegistrations().first) + + #expect(registration.argumentTypes["priority"] == .string) + #expect(registration.argumentConstraints.allowedStringValues["priority"] == ["low", "normal", "high"]) + // The generated example is a call the model can pattern-match, not the + // vacuous `await path({})` that used to be emitted regardless of arguments. + #expect(registration.example == #"await macro.api.prioritize({ id: "id" })"#) + // Tags cover every path segment, so the helper is findable by namespace and + // by name, not only by the full parent path. + #expect(registration.tags.contains("prioritize")) + #expect(registration.tags.contains("macro.api")) +} + +@Test func codeModeMacroRejectsValuesOutsideTheConstraint() async throws { + let tools = CodeModeAgentTools(config: CodeModeConfiguration(codeModeProviders: [PrioritizeTool()])) + + let denied = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await macro.api.prioritize({ id: "1", priority: "urgent" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["macro.api.prioritize"] + ) + ) + var deniedError: CodeModeToolError? + do { + _ = try await denied.result + } catch let error as CodeModeToolError { + deniedError = error + } + #expect(deniedError?.code == "INVALID_ARGUMENTS") + + let accepted = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await macro.api.prioritize({ id: "1", priority: "high" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["macro.api.prioritize"] + ) + ) + let result = try await accepted.result + #expect(result.output?.objectValue?["priority"] == .string("high")) +} diff --git a/Tests/CodeModeTests/CodeModeProviderTests.swift b/Tests/CodeModeTests/CodeModeProviderTests.swift index 75641a9..e444951 100644 --- a/Tests/CodeModeTests/CodeModeProviderTests.swift +++ b/Tests/CodeModeTests/CodeModeProviderTests.swift @@ -705,3 +705,78 @@ private func jsonLiteral(_ value: JSONValue) -> String { #expect(result.string("custom") == "myapp.api.doTheThing") #expect(result.string("builtIn") == CapabilityID.weatherRead.rawValue) } + +// MARK: - Post-init provider registration + +private struct LateProvider: CodeModeProvider { + let codeModePath = "myapp.late" + + func codeModeRegistrations() -> [CodeModeRegistration] { + [ + CodeModeRegistration( + capabilityKey: "myapp.late.echo", + jsPath: "myapp.late.echo", + title: "echo", + summary: "Echo a value registered after init.", + tags: ["myapp", "late"], + example: #"await myapp.late.echo({ value: "hi" })"#, + requiredArguments: ["value"], + argumentTypes: ["value": .string], + argumentHints: ["value": "Value to echo"], + resultSummary: "Echoed value" + ) { arguments, _ in + .object(["value": .string(try CodeModeArgumentDecoder.requireString("value", in: arguments))]) + }, + ] + } +} + +// The catalog used to be snapshotted in CodeModeAgentTools.init, so the +// registry's public post-init register(...) methods were unreachable in the +// supported flow — and would have desynced search from execution if reached. +@Test func providersRegisteredAfterInitAreExecutable() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let before = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.late.echo({ value: "hi" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.late.echo"] + ) + ) + // Not registered yet: the helper simply does not exist in the context. + let beforeObserved = await observe(before) + #expect(beforeObserved.error != nil) + + tools.register(provider: LateProvider()) + + let call = try await tools.executeJavaScript( + JavaScriptExecutionRequest( + code: #"return await myapp.late.echo({ value: "hi" });"#, + allowedCapabilities: [], + allowedCapabilityKeys: ["myapp.late.echo"] + ) + ) + let observed = await observe(call) + #expect(observed.result?.output?.objectValue?.string("value") == "hi") +} + +@Test func providersRegisteredAfterInitAppearInSearchAndTypeDeclarations() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + #expect(tools.capabilities().contains { $0.capability == "myapp.late.echo" } == false) + + tools.register(provider: LateProvider()) + + // Search must not advertise anything execution cannot invoke, and vice versa. + let reference = try #require(tools.capabilities().first { $0.capability == "myapp.late.echo" }) + #expect(reference.dts.contains("value: string;")) + #expect(tools.typeDeclarations().contains("namespace late {")) + + let response = try await tools.searchJavaScriptAPI( + JavaScriptAPISearchRequest(code: #"async () => { return api.byJSName["myapp.late.echo"].summary; }"#) + ) + #expect(response.result == .string("Echo a value registered after init.")) +} From c7e7a5d6bb500c341ae049457680224bf68f6473 Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 20:51:37 -0700 Subject: [PATCH 14/18] Cover the EventKit serialization crashes, pin CI, record what is left MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EventKit's serialization logic executed nowhere — not in unit tests, not in CI, not in evals — because the bridge hard-instantiates `EKEventStore` and the tests can only reach permission denial. The two implicitly-unwrapped fields that crashed the execution thread are now covered directly by constructing the model objects, which needs no permission: an `EKEvent` with no calendar and an `EKReminder` with no title. CI runners are pinned to `macos-15` rather than `macos-latest`, so the package's declared macOS 15 floor cannot drift without a code change. TODO.md gains a "review follow-up — still open" section listing every part of REVIEW.md's P1 items that was not completed — the async handler signature and concurrent Promise.all, native-call interruption, generating the JS bootstrap, `@CodeModeProvider`/`@CodeModeTool`, the MCP adapter, the remaining bridge store seams, a simulator job, and the "prompt pending" permission status — so the finished parts are not mistaken for the whole. It also records that the review's claim about missing `FetchTaskHandler` redirect and size-cap coverage was stale; both are covered in NetworkAccessPolicyTests. --- .github/workflows/codemode-evals.yml | 8 ++- Sources/CodeMode/Bridges/EventKitBridge.swift | 11 ++++ TODO.md | 59 +++++++++++++++++++ Tests/CodeModeTests/EventKitBridgeTests.swift | 41 +++++++++++++ 4 files changed, 116 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codemode-evals.yml b/.github/workflows/codemode-evals.yml index 18c7eda..05a4ed9 100644 --- a/.github/workflows/codemode-evals.yml +++ b/.github/workflows/codemode-evals.yml @@ -26,7 +26,9 @@ jobs: # unchanged main added no signal for these offline checks. deterministic: name: Deterministic evals - runs-on: macos-latest + # Pinned, not macos-latest: the package declares a macOS 15 floor, and a + # silently-rolling runner would move that floor without a code change. + runs-on: macos-15 steps: - name: Checkout uses: actions/checkout@v4 @@ -57,7 +59,7 @@ jobs: platform-build: name: Build (${{ matrix.platform }}) - runs-on: macos-latest + runs-on: macos-15 strategy: fail-fast: false matrix: @@ -107,7 +109,7 @@ jobs: # live+compare steps with a warning, instead of pretending or failing noisily. llm-live: name: LLM eval (live regression gate) - runs-on: macos-latest + runs-on: macos-15 needs: deterministic if: github.event_name == 'workflow_dispatch' env: diff --git a/Sources/CodeMode/Bridges/EventKitBridge.swift b/Sources/CodeMode/Bridges/EventKitBridge.swift index b0b6a98..9750c4a 100644 --- a/Sources/CodeMode/Bridges/EventKitBridge.swift +++ b/Sources/CodeMode/Bridges/EventKitBridge.swift @@ -491,6 +491,17 @@ public final class EventKitBridge: @unchecked Sendable { return span.ekSpan } + /// Test seam: these serializers are the only EventKit logic reachable + /// without a live, permission-granted store, and they are where the + /// implicitly-unwrapped `calendar`/`title` crashes lived. + static func eventJSONForTesting(_ event: EKEvent) -> JSONValue { + eventJSON(event) + } + + static func reminderJSONForTesting(_ reminder: EKReminder) -> JSONValue { + reminderJSON(reminder) + } + private static func eventJSON(_ event: EKEvent) -> JSONValue { .object([ "identifier": .string(event.eventIdentifier ?? ""), diff --git a/TODO.md b/TODO.md index c7516b2..8a3b8e6 100644 --- a/TODO.md +++ b/TODO.md @@ -240,6 +240,65 @@ tool model). → "resource- and network-bounded runtime." 3. Metadata-consolidation refactor + macro decision, protected by new drift tests. +## REVIEW FOLLOW-UP — STILL OPEN +From `REVIEW.md`. Everything in P0 and P2 is done; these are the parts of the P1 +items that were not completed, kept explicit so they do not read as finished. + +### #9 — async bridge ABI (partially done) +Done: real timer queue (`setTimeout`/`clearTimeout` honour delays and cancel), +immediate diagnosis of unsettleable promises, `BRIDGE_FAILURES_NOT_SURFACED` for +swallowed rejections, bounded concurrent executions, `ContinuousClock` deadlines, +deinit cancellation. +- [ ] **`CapabilityHandler` is still synchronous**, so `Promise.all([fetch(a), fetch(b)])` + is still serial. This is the change that requires resolving JS promises from + Swift via retained resolve/reject `JSValue`s on a per-execution serial executor. +- [ ] **Timeout/cancel still cannot interrupt an in-flight native call.** The + watchdog only traps while JS runs; `MediaBridge.transcode` can still overrun a + short `timeoutMs` by minutes. +- [ ] **`CodeModeAsyncBridge.run` still parks a semaphore** for up to 30s (longer + than the 10s default execution timeout) for async host tools. +- [ ] Optional: pool `JSVirtualMachine`s. Constraint to respect — the JSC time + limit is per context *group*, so a pooled VM must never host two concurrent + executions. + +### #11 — generate the JS bindings from the catalog +Done: the concrete drift found while generating declarations is fixed (keychain +object/positional, `fs.promises.rename`/`copyFile` options, location argument +pass-through, tool-description calling conventions). +- [ ] `RuntimeJavaScript.bootstrap` is still ~250 hand-written lines that can + drift from the catalog again. Wrappers that inject hidden arguments + (`createEvent` forces `operation: 'create'`) still cannot be expressed in + catalog metadata, so the generated `.d.ts` advertises an argument the wrapper + overrides. + +### #13 — provider ergonomics + MCP +Done: catalog unfrozen (post-init `register(provider:)` works and stays in sync), +enum constraints, real examples, per-segment tags. +- [ ] `@CodeModeProvider(namespace:)` on a type with `@CodeModeTool` on methods — + today it is still one struct per JavaScript function. +- [ ] `requiredPermissions` and typed results for provider registrations. +- [ ] MCP adapter materializing an MCP server's tools as `CodeModeRegistration`s + under `mcp..`. Blocked on async handlers (#9). + +### Test seams +Done: EventKit event/reminder serialization is now covered directly (the two +implicit-unwrap crashes), concurrent execution is covered, CI runners are pinned +to `macos-15` rather than `macos-latest`. `FetchTaskHandler`'s redirect +re-validation and size-cap cancellation turned out to be covered already +(`NetworkAccessPolicyTests`), contrary to the review. +- [ ] **Store protocols for EventKit/Health/Photos/Home/Contacts**, mirroring + `CodeModeFileSystem`. Their create/update/filter logic still executes nowhere: + the bridges hard-instantiate `EKEventStore()`, `HKHealthStore()`, `PHAsset`, so + tests can only reach permission denial and argument validation. +- [ ] A simulator test job, so UIKit presenters, AlarmKit, and the TCC-gated happy + paths are executed somewhere rather than only compiled. + +### Smaller P2 items not taken +- [ ] **No "prompt pending" permission status.** `request*` waits 10s and ignores + the timeout, so a user who takes 11s gets `PERMISSION_DENIED` while the dialog + is still on screen. Needs a new `PermissionStatus` case, which is source-breaking + for host code switching exhaustively. + ## DECLINED REVIEW FINDINGS - **"Stop imposing swift-syntax on every consumer" (REVIEW.md P1 #12)** — considered 2026-07-25 and declined. The proposal is to check in the expanded diff --git a/Tests/CodeModeTests/EventKitBridgeTests.swift b/Tests/CodeModeTests/EventKitBridgeTests.swift index 6fb95b9..da6c32a 100644 --- a/Tests/CodeModeTests/EventKitBridgeTests.swift +++ b/Tests/CodeModeTests/EventKitBridgeTests.swift @@ -135,3 +135,44 @@ import Testing #expect(observed.error?.code == "PERMISSION_DENIED") } + +// MARK: - Serialization of degenerate items + +#if canImport(EventKit) +import EventKit + +// EventKit's serialization logic executes nowhere today — not in unit tests, not +// in CI, not in evals — because the bridge hard-instantiates EKEventStore and the +// tests can only reach permission denial. These construct the model objects +// directly, which needs no permission, and cover the two implicitly-unwrapped +// fields that crashed the execution thread. + +@Test func eventSerializationSurvivesAnEventWithNoCalendar() throws { + let store = EKEventStore() + let event = EKEvent(eventStore: store) + event.title = "Orphaned" + event.startDate = Date(timeIntervalSince1970: 1_700_000_000) + event.endDate = event.startDate.addingTimeInterval(3_600) + + // `EKEvent.calendar` is `EKCalendar!` and is nil for an orphaned event, while + // every neighbouring field was nil-coalesced. + #expect(event.calendar == nil) + + let json = try #require(EventKitBridge.eventJSONForTesting(event).objectValue) + #expect(json.string("title") == "Orphaned") + #expect(json.string("calendarIdentifier") == "") + #expect(json.string("calendarTitle") == "") + #expect(json.string("notes") == "") +} + +@Test func reminderSerializationSurvivesAMissingTitle() throws { + let store = EKEventStore() + let reminder = EKReminder(eventStore: store) + + // `EKReminder.title` is `String!`. + let json = try #require(EventKitBridge.reminderJSONForTesting(reminder).objectValue) + #expect(json.string("title") == "") + #expect(json.string("calendarIdentifier") == "") + #expect(json.bool("isCompleted") == false) +} +#endif From 3cfcc919d3c145f54ef87250cfae14c89bf3b77f Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 21:04:31 -0700 Subject: [PATCH 15/18] Simplify the review-fix code: dedupe, drop dead code, tighten seams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Quality pass over this branch's changes. No behavior change except one bug the refactor exposed, noted below. Reuse: - One `BridgeInvocationContext.toolError(...)` replaces three separate places assembling `CodeModeToolError` from the same three transcript accessors — the runtime's `toolError`, `SettlementParameters.error`, and `terminationError`. It belongs on the context, which owns those fields. - One `stringArray(from:evaluating:)` replaces the duplicated `JSON.stringify` → decode `[String]` round-trip in `rejectedSuggestions` and the timer-error read. - One `FileSystemBridge.overLimit(...)` replaces three hand-built size-limit messages that had already drifted apart in wording. Simplification: - `SettlementParameters` bundles the six values `waitForSettlement` and `runDueTimers` both needed; the two signatures drop from 7 and 8 parameters to 2 and 3. `checkInterrupted` replaces the cancellation/termination/deadline ladder both were repeating. - The host-withheld denial branch is hoisted above the `switch` instead of being nested identically inside two cases. - `decodeOutput` checks the watchdog once rather than side by side in a guard's else and again after it; its embedded JS returns the in-range value from one place instead of two. - `ExecutionTranscript.record(log:)` now matches the shape of `record(diagnostic:)` — one lock/unlock, no branch-local unlocking — and drops `droppedLogs`, which was incremented and never read. - `TypeScriptDeclarations.functionSignature` drops an unused `name:` parameter, and `lastComponent(of:)`, which existed only to supply it. - `CapabilityGrant.Resolution` keeps `withheld: [String]` instead of two sets that no caller read except through the accessor that joined them. - Dropped `CodeModeDate.string(from:)` — never called. Efficiency: - `advanceTimers` returns its callbacks' uncaught errors instead of buffering them for a second `takeTimerErrors` call, halving the bridge round-trips per timer tick. - The elapsed-time computation reads the clock once through a `Duration` extension rather than twice through a two-term attosecond conversion, and drops a `max` that could not fire. Bug found while refactoring: bundling the interrupt checks moved the deadline check ahead of the settled-state read, which is exactly the spurious-timeout regression the original comment warned about — a script that fulfilled a hair past the deadline would have had its result discarded. Ordering restored (cancellation before the read, deadline after) and pinned by a new test, which nothing covered before. --- .../CodeMode/Bridges/FileSystemBridge.swift | 35 +- .../Catalog/TypeScriptDeclarations.swift | 17 +- .../Runtime/BridgeInvocationContext.swift | 26 ++ Sources/CodeMode/Runtime/BridgeRuntime.swift | 299 +++++++++--------- .../CodeMode/Runtime/ExecutionSupport.swift | 42 ++- .../CodeMode/Runtime/RuntimeJavaScript.swift | 13 +- .../CodeMode/Security/CapabilityGrant.swift | 15 +- .../Security/SystemPermissionBroker.swift | 18 +- Sources/CodeMode/Support/CodeModeDate.swift | 4 - Tests/CodeModeTests/EventLoopTests.swift | 26 ++ 10 files changed, 263 insertions(+), 232 deletions(-) diff --git a/Sources/CodeMode/Bridges/FileSystemBridge.swift b/Sources/CodeMode/Bridges/FileSystemBridge.swift index 4ba8b25..1118038 100644 --- a/Sources/CodeMode/Bridges/FileSystemBridge.swift +++ b/Sources/CodeMode/Bridges/FileSystemBridge.swift @@ -63,20 +63,12 @@ public final class FileSystemBridge: @unchecked Sendable { let encoding = arguments.string("encoding") ?? "utf8" let url = try context.pathPolicy.resolve(path: path) - // Check the size before reading: reading first and then rejecting has - // already spent the memory the cap exists to protect. - if let size = try? fileSystem.attributesOfItem(at: url).size, size > limits.maxReadBytes { - throw BridgeError.invalidArguments( - "fs.read refused \(url.lastPathComponent): \(size) bytes exceeds the \(limits.maxReadBytes)-byte read limit. Read a smaller file, or have the host raise CodeModeConfiguration.fileSystemLimits." - ) - } - + // Checked against stat first: reading and then rejecting has already spent + // the memory the cap exists to protect. Re-checked after, because a + // custom CodeModeFileSystem's stat may be absent or stale. + try enforceReadLimit(try? fileSystem.attributesOfItem(at: url).size, at: url) let data = try fileSystem.readData(at: url) - guard data.count <= limits.maxReadBytes else { - throw BridgeError.invalidArguments( - "fs.read refused \(url.lastPathComponent): \(data.count) bytes exceeds the \(limits.maxReadBytes)-byte read limit." - ) - } + try enforceReadLimit(data.count, at: url) switch encoding.lowercased() { case "utf8", "utf-8": @@ -124,9 +116,7 @@ public final class FileSystemBridge: @unchecked Sendable { } guard data.count <= limits.maxWriteBytes else { - throw BridgeError.invalidArguments( - "fs.write refused \(url.lastPathComponent): \(data.count) bytes exceeds the \(limits.maxWriteBytes)-byte write limit." - ) + throw Self.overLimit("fs.write", url: url, bytes: data.count, limit: limits.maxWriteBytes) } // Only after the payload is known good, so a rejected write leaves no @@ -171,6 +161,19 @@ public final class FileSystemBridge: @unchecked Sendable { ]) } + private func enforceReadLimit(_ bytes: Int?, at url: URL) throws { + guard let bytes, bytes > limits.maxReadBytes else { + return + } + throw Self.overLimit("fs.read", url: url, bytes: bytes, limit: limits.maxReadBytes) + } + + private static func overLimit(_ operation: String, url: URL, bytes: Int, limit: Int) -> BridgeError { + .invalidArguments( + "\(operation) refused \(url.lastPathComponent): \(bytes) bytes exceeds the \(limit)-byte limit. Use a smaller payload, or have the host raise CodeModeConfiguration.fileSystemLimits." + ) + } + /// Clears the way for a move/copy, refusing anything destructive the caller /// did not explicitly ask for. /// diff --git a/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift b/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift index 81e891d..6edd257 100644 --- a/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift +++ b/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift @@ -91,8 +91,8 @@ public enum TypeScriptDeclarations { guard let canonical = canonicalName(for: reference) else { return "" } - let signature = functionSignature(for: reference, name: lastComponent(of: canonical)) - return "\(documentation(for: reference, canonicalName: canonical))\ndeclare function \(canonical.replacingOccurrences(of: ".", with: "_"))\(signature);" + let name = canonical.replacingOccurrences(of: ".", with: "_") + return "\(documentation(for: reference, canonicalName: canonical))\ndeclare function \(name)\(functionSignature(for: reference));" } /// The whole surface: the preamble plus every capability, grouped into @@ -113,7 +113,7 @@ public enum TypeScriptDeclarations { Member( name: functionName, documentation: documentation(for: reference, canonicalName: canonical), - signature: functionSignature(for: reference, name: functionName) + signature: functionSignature(for: reference) ), at: components ) @@ -144,10 +144,6 @@ public enum TypeScriptDeclarations { ?? reference.jsNames.first } - private static func lastComponent(of name: String) -> String { - name.split(separator: ".").last.map(String.init) ?? name - } - private static func documentation(for reference: JavaScriptAPIReference, canonicalName: String) -> String { var lines = ["/**"] lines.append(" * \(reference.summary)") @@ -171,13 +167,14 @@ public enum TypeScriptDeclarations { return lines.joined(separator: "\n") } - private static func functionSignature(for reference: JavaScriptAPIReference, name: String) -> String { + private static func functionSignature(for reference: JavaScriptAPIReference) -> String { let tree = ArgumentTree(reference: reference) guard tree.isEmpty == false else { return "(): Promise" } - let hasRequired = reference.requiredArguments.isEmpty == false - return "(args\(hasRequired ? "" : "?"): \(tree.render(indent: "")))" + ": Promise" + // `args` is optional only when nothing in it is required. + let optionalMarker = reference.requiredArguments.isEmpty ? "?" : "" + return "(args\(optionalMarker): \(tree.render(indent: ""))): Promise" } // MARK: - Argument tree diff --git a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift index 2d3ff06..1dc9cc2 100644 --- a/Sources/CodeMode/Runtime/BridgeInvocationContext.swift +++ b/Sources/CodeMode/Runtime/BridgeInvocationContext.swift @@ -115,6 +115,32 @@ public final class BridgeInvocationContext: @unchecked Sendable { hostWithheldCapabilityIdentifiers.contains(identifier) } + /// Builds a `CodeModeToolError` carrying this execution's transcript. + /// + /// Lives here rather than on the runtime because the transcript fields are + /// this type's; three separate call sites were otherwise assembling the same + /// three accessors by hand. + func toolError( + code: String, + message: String, + functionName: String? = nil, + capability: CapabilityID? = nil, + capabilityKey: CodeModeCapabilityKey? = nil, + suggestions: [String] = [] + ) -> CodeModeToolError { + CodeModeToolError( + code: code, + message: message, + functionName: functionName, + capability: capability, + capabilityKey: capabilityKey, + suggestions: suggestions, + diagnostics: allDiagnostics(), + logs: allLogs(), + permissionEvents: allPermissionEvents() + ) + } + func recordDiagnostic(_ diagnostic: ToolDiagnostic) { transcript.record(diagnostic: diagnostic) } diff --git a/Sources/CodeMode/Runtime/BridgeRuntime.swift b/Sources/CodeMode/Runtime/BridgeRuntime.swift index 8b583b2..5691521 100644 --- a/Sources/CodeMode/Runtime/BridgeRuntime.swift +++ b/Sources/CodeMode/Runtime/BridgeRuntime.swift @@ -214,7 +214,7 @@ final class BridgeRuntime: @unchecked Sendable { requestedCapabilityKeys: Set(request.allowedCapabilityKeys) ) - let withheld = grant.withheldEverything + let withheld = grant.withheld if withheld.isEmpty == false { transcript.record( diagnostic: ToolDiagnostic( @@ -496,16 +496,14 @@ final class BridgeRuntime: @unchecked Sendable { switch watchdog.termination { case .timedOut: - throw toolError( + throw invocationContext.toolError( code: "EXECUTION_TIMEOUT", - message: "Execution timed out after \(timeoutMs)ms", - transcript: invocationContext + message: "Execution timed out after \(timeoutMs)ms" ) case .cancelled: - throw toolError( + throw invocationContext.toolError( code: "CANCELLED", - message: "Execution cancelled", - transcript: invocationContext + message: "Execution cancelled" ) case nil: break @@ -518,12 +516,14 @@ final class BridgeRuntime: @unchecked Sendable { let settlement = try waitForSettlement( context: context, - watchdog: watchdog, - cancellationController: cancellationController, - invocationContext: invocationContext, - timeoutCode: "EXECUTION_TIMEOUT", - timeoutMessage: "Execution timed out after \(timeoutMs)ms", - cancelMessage: "Execution cancelled" + settlement: SettlementParameters( + watchdog: watchdog, + cancellationController: cancellationController, + invocationContext: invocationContext, + timeoutCode: "EXECUTION_TIMEOUT", + timeoutMessage: "Execution timed out after \(timeoutMs)ms", + cancelMessage: "Execution cancelled" + ) ) // Give result serialization its own bounded budget so a script that @@ -572,21 +572,20 @@ final class BridgeRuntime: @unchecked Sendable { /// never settle instead of sleeping a thread to the deadline for nothing. private func waitForSettlement( context: JSContext, - watchdog: ExecutionWatchdog, - cancellationController: ExecutionCancellationController, - invocationContext: BridgeInvocationContext, - timeoutCode: String, - timeoutMessage: String, - cancelMessage: String + settlement: SettlementParameters ) throws -> SettlementState { while true { - // `Task.isCancelled` is not checked here: this runs on a plain GCD - // worker with no surrounding Task, so it was always false — dead code + // Cancellation only, deliberately: the deadline is checked *after* + // the settled state below, so a script that fulfilled a hair past it + // still returns its result instead of a spurious timeout. + // + // `Task.isCancelled` is not checked at all — this runs on a plain GCD + // worker with no surrounding Task, so it was always false, dead code // that read as a second layer of cancellation support. The // controller, which `JavaScriptExecutionCall.cancel()` sets, is the // real signal. - if cancellationController.isCancelled { - throw toolError(code: "CANCELLED", message: cancelMessage, transcript: invocationContext) + if settlement.cancellationController.isCancelled { + throw settlement.error(code: "CANCELLED", message: settlement.cancelMessage) } let state = context.evaluateScript("globalThis.__codemode.state")?.toString() ?? "unknown" @@ -596,17 +595,7 @@ final class BridgeRuntime: @unchecked Sendable { case "rejected": return .rejected default: - if let termination = watchdog.termination { - switch termination { - case .cancelled: - throw toolError(code: "CANCELLED", message: cancelMessage, transcript: invocationContext) - case .timedOut: - throw toolError(code: timeoutCode, message: timeoutMessage, transcript: invocationContext) - } - } - if watchdog.hasPassedDeadline { - throw toolError(code: timeoutCode, message: timeoutMessage, transcript: invocationContext) - } + try checkInterrupted(settlement) let nextTimerDelay = context .evaluateScript("globalThis.__codemode.nextTimerDelay()")? @@ -614,10 +603,9 @@ final class BridgeRuntime: @unchecked Sendable { .doubleValue ?? -1 guard nextTimerDelay >= 0 else { - throw toolError( + throw settlement.error( code: "JS_RUNTIME_ERROR", message: "Script finished with a promise that can never settle: nothing is left to resolve it. This runtime has no I/O event loop, so only a pending setTimeout can advance a program after the synchronous work is done.", - transcript: invocationContext, suggestions: [ "Await every promise the script creates, and make sure each one has a resolve path.", "A `new Promise(...)` whose executor never calls resolve or reject will hang forever here.", @@ -625,68 +613,74 @@ final class BridgeRuntime: @unchecked Sendable { ) } - try runDueTimers( - delayMs: nextTimerDelay, - context: context, - watchdog: watchdog, - cancellationController: cancellationController, - invocationContext: invocationContext, - timeoutCode: timeoutCode, - timeoutMessage: timeoutMessage, - cancelMessage: cancelMessage - ) + try runDueTimers(delayMs: nextTimerDelay, context: context, settlement: settlement) } } } + /// Everything the settlement loop needs to report a cancellation or timeout. + /// The execution and search paths differ only in these message strings. + private struct SettlementParameters { + var watchdog: ExecutionWatchdog + var cancellationController: ExecutionCancellationController + var invocationContext: BridgeInvocationContext + var timeoutCode: String + var timeoutMessage: String + var cancelMessage: String + + func error(code: String, message: String, suggestions: [String] = []) -> CodeModeToolError { + invocationContext.toolError(code: code, message: message, suggestions: suggestions) + } + } + + /// Throws if the run was cancelled or has run out of time — the check every + /// waiting loop in this file has to make before sleeping again. + private func checkInterrupted(_ settlement: SettlementParameters) throws { + if settlement.cancellationController.isCancelled { + throw settlement.error(code: "CANCELLED", message: settlement.cancelMessage) + } + switch settlement.watchdog.termination { + case .cancelled: + throw settlement.error(code: "CANCELLED", message: settlement.cancelMessage) + case .timedOut: + throw settlement.error(code: settlement.timeoutCode, message: settlement.timeoutMessage) + case nil: + break + } + if settlement.watchdog.hasPassedDeadline { + throw settlement.error(code: settlement.timeoutCode, message: settlement.timeoutMessage) + } + } + /// Sleeps out a timer's delay — bounded by the deadline and responsive to /// cancellation — then fires everything that has come due. private func runDueTimers( delayMs: Double, context: JSContext, - watchdog: ExecutionWatchdog, - cancellationController: ExecutionCancellationController, - invocationContext: BridgeInvocationContext, - timeoutCode: String, - timeoutMessage: String, - cancelMessage: String + settlement: SettlementParameters ) throws { let start = ContinuousClock.now let target = start.advanced(by: .milliseconds(Int(delayMs.rounded(.up)))) while ContinuousClock.now < target { - if cancellationController.isCancelled { - throw toolError(code: "CANCELLED", message: cancelMessage, transcript: invocationContext) - } - if watchdog.hasPassedDeadline { - throw toolError(code: timeoutCode, message: timeoutMessage, transcript: invocationContext) - } + try checkInterrupted(settlement) // Short slices so cancellation and the deadline stay responsive // during a long backoff. Thread.sleep(forTimeInterval: 0.005) } - let elapsed = Double((ContinuousClock.now - start).components.attoseconds) / 1e15 - + Double((ContinuousClock.now - start).components.seconds) * 1_000 - _ = context.evaluateScript("globalThis.__codemode.advanceTimers(\(max(elapsed, delayMs)))") - - recordTimerErrors(from: context, invocationContext: invocationContext) - } - - /// An error thrown by a timer callback cannot propagate to whoever called - /// `setTimeout`, so it surfaces as a diagnostic the way an uncaught task - /// error does in a real event loop. - private func recordTimerErrors(from context: JSContext, invocationContext: BridgeInvocationContext) { - guard let json = context.evaluateScript("JSON.stringify(globalThis.__codemode.takeTimerErrors())")?.toString(), - let data = json.data(using: .utf8), - let messages = try? JSONDecoder.codeModeBridge.decode([String].self, from: data), - messages.isEmpty == false - else { - return - } + // Advance by the time that actually passed, so timers that came due while + // we slept fire together. The call returns the callbacks' uncaught errors, + // which cannot propagate to whoever called `setTimeout` and so surface as + // diagnostics the way an uncaught task error does in a real event loop. + let elapsed = (ContinuousClock.now - start).milliseconds + let errors = Self.stringArray( + from: context, + evaluating: "globalThis.__codemode.advanceTimers(\(elapsed))" + ) - for message in messages { - invocationContext.recordDiagnostic( + for message in errors { + settlement.invocationContext.recordDiagnostic( ToolDiagnostic( severity: .warning, code: "TIMER_CALLBACK_ERROR", @@ -807,16 +801,14 @@ final class BridgeRuntime: @unchecked Sendable { switch watchdog.termination { case .timedOut: - throw toolError( + throw invocationContext.toolError( code: "SEARCH_TIMEOUT", - message: "Search timed out after \(timeoutMs)ms", - transcript: invocationContext + message: "Search timed out after \(timeoutMs)ms" ) case .cancelled: - throw toolError( + throw invocationContext.toolError( code: "CANCELLED", - message: "Search cancelled", - transcript: invocationContext + message: "Search cancelled" ) case nil: break @@ -829,12 +821,14 @@ final class BridgeRuntime: @unchecked Sendable { let settlement = try waitForSettlement( context: context, - watchdog: watchdog, - cancellationController: cancellationController, - invocationContext: invocationContext, - timeoutCode: "SEARCH_TIMEOUT", - timeoutMessage: "Search timed out after \(timeoutMs)ms", - cancelMessage: "Search cancelled" + settlement: SettlementParameters( + watchdog: watchdog, + cancellationController: cancellationController, + invocationContext: invocationContext, + timeoutCode: "SEARCH_TIMEOUT", + timeoutMessage: "Search timed out after \(timeoutMs)ms", + cancelMessage: "Search cancelled" + ) ) watchdog.rearm(timeoutMs: min(timeoutMs, Self.serializationBudgetMs)) @@ -871,15 +865,12 @@ final class BridgeRuntime: @unchecked Sendable { // already spent the memory the cap exists to protect. The replacement is // still valid JSON, so the model gets a parseable value plus a // diagnostic rather than a truncated fragment it cannot read. - guard let serialization = context.evaluateScript( + let serialization = context.evaluateScript( """ (function(){ try { var json = JSON.stringify(globalThis.__codemode.result); - if (typeof json !== 'string') { - return { ok: true, json: json }; - } - if (json.length > \(limits.maxResultCharacters)) { + if (typeof json === 'string' && json.length > \(limits.maxResultCharacters)) { return { ok: true, truncated: true, @@ -898,20 +889,18 @@ final class BridgeRuntime: @unchecked Sendable { } })() """ - ) else { - // A nil evaluation here after the watchdog fired is a termination, - // not a serialization problem. Reporting `INVALID_RESULT: must be - // JSON-serializable` for a runaway getter steered the model to fix - // the wrong thing. - if let termination = watchdog?.termination { - throw terminationError(termination, invocationContext: invocationContext, during: "result serialization") - } - throw CodeModeToolError(code: errorCode, message: errorMessagePrefix) - } + ) + // Checked before anything else: a watchdog termination during + // serialization is a timeout, not a serialization problem, and reporting + // `INVALID_RESULT: must be JSON-serializable` for a runaway getter + // steered the model to fix the wrong thing. if let termination = watchdog?.termination { throw terminationError(termination, invocationContext: invocationContext, during: "result serialization") } + guard let serialization else { + throw CodeModeToolError(code: errorCode, message: errorMessagePrefix) + } if serialization.forProperty("ok")?.toBool() == false { let message = serialization.forProperty("message")?.toString() ?? errorMessagePrefix @@ -962,10 +951,8 @@ final class BridgeRuntime: @unchecked Sendable { ("CANCELLED", "Execution cancelled during \(phase)") } - guard let invocationContext else { - return CodeModeToolError(code: code, message: message) - } - return toolError(code: code, message: message, transcript: invocationContext) + return invocationContext?.toolError(code: code, message: message) + ?? CodeModeToolError(code: code, message: message) } private func bridgeFailurePayload( @@ -1101,13 +1088,19 @@ final class BridgeRuntime: @unchecked Sendable { } private func rejectedSuggestions(from context: JSContext) -> [String] { - guard let json = context.evaluateScript("JSON.stringify(globalThis.__codemode.error?.suggestions ?? [])")?.toString(), + Self.stringArray(from: context, evaluating: "globalThis.__codemode.error?.suggestions ?? []") + } + + /// Reads a JavaScript array of strings across the bridge. Returns an empty + /// array if the expression is missing or not string-shaped. + private static func stringArray(from context: JSContext, evaluating expression: String) -> [String] { + guard let json = context.evaluateScript("JSON.stringify(\(expression))")?.toString(), let data = json.data(using: .utf8), - let suggestions = try? JSONDecoder.codeModeBridge.decode([String].self, from: data) + let values = try? JSONDecoder.codeModeBridge.decode([String].self, from: data) else { return [] } - return suggestions + return values } private func event(for error: CodeModeToolError) -> JavaScriptExecutionEvent { @@ -1123,28 +1116,6 @@ final class BridgeRuntime: @unchecked Sendable { } } - private func toolError( - code: String, - message: String, - transcript: BridgeInvocationContext, - functionName: String? = nil, - capability: CapabilityID? = nil, - capabilityKey: CodeModeCapabilityKey? = nil, - suggestions: [String] = [] - ) -> CodeModeToolError { - CodeModeToolError( - code: code, - message: message, - functionName: functionName, - capability: capability, - capabilityKey: capabilityKey, - suggestions: suggestions, - diagnostics: transcript.allDiagnostics(), - logs: transcript.allLogs(), - permissionEvents: transcript.allPermissionEvents() - ) - } - private func bridgeSuggestions(for capability: CapabilityID?) -> [String] { bridgeSuggestions(for: capability, capabilityKey: capability?.codeModeKey) } @@ -1179,26 +1150,37 @@ final class BridgeRuntime: @unchecked Sendable { capabilityKey: CodeModeCapabilityKey?, invocationContext: BridgeInvocationContext? = nil ) -> (String, [String]) { + // A host-withheld denial reads the same whichever allowlist it came from, + // and is the one denial the model cannot repair — so it short-circuits + // both cases below. + let deniedIdentifier: String? = switch error { + case let .capabilityDenied(capability): capability.rawValue + case let .capabilityKeyDenied(capabilityKey): capabilityKey.rawValue + default: nil + } + if let deniedIdentifier, invocationContext?.isWithheldByHost(deniedIdentifier) == true { + return ( + error.localizedDescription, + [ + "The host app's capability grant does not include \"\(deniedIdentifier)\".", + "This is not repaired by adding it to allowedCapabilities or allowedCapabilityKeys; the host owns this decision.", + "Continue with the capabilities you do have, or report the missing access to the user.", + ] + ) + } + let suggestions: [String] switch error { case let .capabilityDenied(capability): - if invocationContext?.isWithheldByHost(capability.rawValue) == true { - suggestions = hostWithheldSuggestions(for: capability.rawValue) - } else { - suggestions = [ - "Add \"\(capability.rawValue)\" to allowedCapabilities and retry.", - "Built-in capabilities are granted only by allowedCapabilities; listing one in allowedCapabilityKeys has no effect.", - ] + bridgeSuggestions(for: capability, capabilityKey: capability.codeModeKey) - } + suggestions = [ + "Add \"\(capability.rawValue)\" to allowedCapabilities and retry.", + "Built-in capabilities are granted only by allowedCapabilities; listing one in allowedCapabilityKeys has no effect.", + ] + bridgeSuggestions(for: capability, capabilityKey: capability.codeModeKey) case let .capabilityKeyDenied(capabilityKey): - if invocationContext?.isWithheldByHost(capabilityKey.rawValue) == true { - suggestions = hostWithheldSuggestions(for: capabilityKey.rawValue) - } else { - suggestions = [ - "Add \"\(capabilityKey.rawValue)\" to allowedCapabilityKeys and retry.", - "Custom provider capabilities are not enabled by allowedCapabilities.", - ] + bridgeSuggestions(for: nil, capabilityKey: capabilityKey) - } + suggestions = [ + "Add \"\(capabilityKey.rawValue)\" to allowedCapabilityKeys and retry.", + "Custom provider capabilities are not enabled by allowedCapabilities.", + ] + bridgeSuggestions(for: nil, capabilityKey: capabilityKey) case let .permissionDenied(permission): suggestions = permissionDeniedSuggestions(for: permission) case .customPermissionDenied: @@ -1224,14 +1206,6 @@ final class BridgeRuntime: @unchecked Sendable { return (error.localizedDescription, suggestions) } - private func hostWithheldSuggestions(for identifier: String) -> [String] { - [ - "The host app's capability grant does not include \"\(identifier)\".", - "This is not repaired by adding it to allowedCapabilities or allowedCapabilityKeys; the host owns this decision.", - "Continue with the capabilities you do have, or report the missing access to the user.", - ] - } - private func permissionDeniedSuggestions(for permission: PermissionKind) -> [String] { var suggestions = [ "Permission \"\(permission.rawValue)\" was denied after capability allowlisting succeeded.", @@ -1388,3 +1362,12 @@ final class BridgeRuntime: @unchecked Sendable { return trimmed } } + +private extension Duration { + /// Whole and fractional milliseconds. `components` splits into seconds plus + /// attoseconds, which is otherwise an awkward two-term conversion. + var milliseconds: Double { + let parts = components + return Double(parts.seconds) * 1_000 + Double(parts.attoseconds) / 1e15 + } +} diff --git a/Sources/CodeMode/Runtime/ExecutionSupport.swift b/Sources/CodeMode/Runtime/ExecutionSupport.swift index 8491f97..4a0db8b 100644 --- a/Sources/CodeMode/Runtime/ExecutionSupport.swift +++ b/Sources/CodeMode/Runtime/ExecutionSupport.swift @@ -23,7 +23,6 @@ final class ExecutionTranscript: @unchecked Sendable { private var logs: [ExecutionLog] = [] private var diagnostics: [ToolDiagnostic] = [] private var permissionEvents: [PermissionEvent] = [] - private var droppedLogs = 0 private var noticedLogOverflow = false private let emitEvent: @Sendable (JavaScriptExecutionEvent) -> Void @@ -39,34 +38,33 @@ final class ExecutionTranscript: @unchecked Sendable { var entry = log entry.message = Self.elide(entry.message, to: limits.maxLogMessageCharacters) - var overflowNotice: ToolDiagnostic? + // Keep the first N: the beginning of a runaway log is where the cause is, + // and the tail is the same line a million times. lock.lock() - if logs.count >= limits.maxLogEntries { - // Keep the first N: the beginning of a runaway log is where the - // cause is, and the tail is the same line a million times. - droppedLogs += 1 - if noticedLogOverflow == false { - noticedLogOverflow = true - overflowNotice = ToolDiagnostic( + let accepted = logs.count < limits.maxLogEntries + let isFirstOverflow = accepted == false && noticedLogOverflow == false + if accepted { + logs.append(entry) + } else { + noticedLogOverflow = true + } + lock.unlock() + + if accepted { + // Streaming stays live even after the retained transcript is full; + // it is the retained copy — re-copied into every CodeModeToolError — + // that is the memory risk. + emitEvent(.log(entry)) + } else if isFirstOverflow { + record( + diagnostic: ToolDiagnostic( severity: .warning, code: "LOG_LIMIT_REACHED", message: "Kept the first \(limits.maxLogEntries) console.log entries; later entries are dropped.", category: "execution", suggestions: ["Log summaries rather than per-item lines, or return the data instead of logging it."] ) - } - lock.unlock() - } else { - logs.append(entry) - lock.unlock() - // Streaming stays live even after the retained transcript is full; - // it is the retained copy — re-copied into every CodeModeToolError — - // that is the memory risk. - emitEvent(.log(entry)) - } - - if let overflowNotice { - record(diagnostic: overflowNotice) + ) } } diff --git a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift index f4d288c..34a7ddc 100644 --- a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift +++ b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift @@ -11,7 +11,7 @@ enum RuntimeJavaScript { // rather than being an uncaught task error, and // `await new Promise(r => setTimeout(r, 2000))` — the standard backoff — was // a hot loop that hammered remote APIs through `fetch`. - globalThis.__codemode.timers = { nextID: 1, entries: [], errors: [] }; + globalThis.__codemode.timers = { nextID: 1, entries: [] }; globalThis.setTimeout = function(fn, delay) { if (typeof fn !== 'function') { @@ -55,21 +55,18 @@ enum RuntimeJavaScript { timers.entries = remaining; // Registration order breaks ties, matching a real event loop. due.sort(function(a, b){ return a.id - b.id; }); + const errors = []; due.forEach(function(entry){ try { entry.fn.apply(null, entry.args); } catch (error) { // An uncaught error in a task cannot propagate to whoever called // setTimeout; the host reports it as a diagnostic instead. - timers.errors.push(String(error)); + errors.push(String(error)); } }); - return due.length; - }; - - globalThis.__codemode.takeTimerErrors = function() { - const errors = globalThis.__codemode.timers.errors; - globalThis.__codemode.timers.errors = []; + // Returned rather than buffered, so the host needs one round trip per + // tick instead of two. return errors; }; """ diff --git a/Sources/CodeMode/Security/CapabilityGrant.swift b/Sources/CodeMode/Security/CapabilityGrant.swift index d88e7c2..af300cd 100644 --- a/Sources/CodeMode/Security/CapabilityGrant.swift +++ b/Sources/CodeMode/Security/CapabilityGrant.swift @@ -58,19 +58,18 @@ public struct CapabilityGrant: Sendable, Equatable { return Resolution( capabilities: effectiveCapabilities, capabilityKeys: effectiveKeys, - withheldCapabilities: requestedCapabilities.subtracting(effectiveCapabilities), - withheldCapabilityKeys: requestedCapabilityKeys.subtracting(effectiveKeys) + withheld: ( + requestedCapabilities.subtracting(effectiveCapabilities).map(\.rawValue) + + requestedCapabilityKeys.subtracting(effectiveKeys).map(\.rawValue) + ).sorted() ) } public struct Resolution: Sendable, Equatable { public var capabilities: Set public var capabilityKeys: Set - public var withheldCapabilities: Set - public var withheldCapabilityKeys: Set - - public var withheldEverything: [String] { - (withheldCapabilities.map(\.rawValue) + withheldCapabilityKeys.map(\.rawValue)).sorted() - } + /// Identifiers the request declared and the grant refused, sorted. Both + /// namespaces together, because every consumer reports them as one list. + public var withheld: [String] } } diff --git a/Sources/CodeMode/Security/SystemPermissionBroker.swift b/Sources/CodeMode/Security/SystemPermissionBroker.swift index cd3b254..36f4510 100644 --- a/Sources/CodeMode/Security/SystemPermissionBroker.swift +++ b/Sources/CodeMode/Security/SystemPermissionBroker.swift @@ -51,6 +51,12 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable /// pre-prompt status. static let permissionPromptTimeoutSeconds: TimeInterval = 10 + /// Every TCC request in this type waits that same budget, so the timeout is + /// not repeated at six call sites where it could drift apart. + private static func awaitingPrompt(_ request: (@escaping @Sendable () -> Void) -> Void) { + CompletionWait.completion(timeout: permissionPromptTimeoutSeconds, request) + } + public init() {} public func status(for permission: PermissionKind) -> PermissionStatus { @@ -509,7 +515,7 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestContactsPermission() -> PermissionStatus { #if canImport(Contacts) let store = CNContactStore() - CompletionWait.completion(timeout: Self.permissionPromptTimeoutSeconds) { complete in + Self.awaitingPrompt { complete in store.requestAccess(for: .contacts) { _, _ in complete() } } return contactsStatus() @@ -521,7 +527,7 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestCalendarPermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - CompletionWait.completion(timeout: Self.permissionPromptTimeoutSeconds) { complete in + Self.awaitingPrompt { complete in if #available(iOS 17.0, macOS 14.0, *) { store.requestFullAccessToEvents { _, _ in complete() } } else { @@ -537,7 +543,7 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestCalendarWritePermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - CompletionWait.completion(timeout: Self.permissionPromptTimeoutSeconds) { complete in + Self.awaitingPrompt { complete in if #available(iOS 17.0, macOS 14.0, *) { store.requestWriteOnlyAccessToEvents { _, _ in complete() } } else { @@ -553,7 +559,7 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestRemindersPermission() -> PermissionStatus { #if canImport(EventKit) let store = EKEventStore() - CompletionWait.completion(timeout: Self.permissionPromptTimeoutSeconds) { complete in + Self.awaitingPrompt { complete in if #available(iOS 17.0, macOS 14.0, *) { store.requestFullAccessToReminders { _, _ in complete() } } else { @@ -568,7 +574,7 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestPhotoLibraryPermission() -> PermissionStatus { #if canImport(Photos) - CompletionWait.completion(timeout: Self.permissionPromptTimeoutSeconds) { complete in + Self.awaitingPrompt { complete in if #available(iOS 14.0, macOS 11.0, *) { PHPhotoLibrary.requestAuthorization(for: .readWrite) { _ in complete() } } else { @@ -583,7 +589,7 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable private func requestNotificationsPermission() -> PermissionStatus { #if canImport(UserNotifications) - CompletionWait.completion(timeout: Self.permissionPromptTimeoutSeconds) { complete in + Self.awaitingPrompt { complete in UNUserNotificationCenter.current().requestAuthorization(options: [.alert, .badge, .sound]) { _, _ in complete() } diff --git a/Sources/CodeMode/Support/CodeModeDate.swift b/Sources/CodeMode/Support/CodeModeDate.swift index a7f3fc8..e3c89d7 100644 --- a/Sources/CodeMode/Support/CodeModeDate.swift +++ b/Sources/CodeMode/Support/CodeModeDate.swift @@ -55,8 +55,4 @@ enum CodeModeDate { } return try require(text, argument: argument, capability: capability) } - - static func string(from date: Date) -> String { - ISO8601DateFormatter().string(from: date) - } } diff --git a/Tests/CodeModeTests/EventLoopTests.swift b/Tests/CodeModeTests/EventLoopTests.swift index 6561b86..25982b3 100644 --- a/Tests/CodeModeTests/EventLoopTests.swift +++ b/Tests/CodeModeTests/EventLoopTests.swift @@ -303,3 +303,29 @@ import Testing #expect(outputs.compactMap { $0?.intValue }.sorted() == [0, 1, 2, 3]) } + +@Test func aResultThatSettlesRightAtTheDeadlineIsNotDiscarded() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // The settled state must be read before the deadline is enforced. Checking + // the deadline first turns a script that fulfilled a hair late into a + // spurious timeout, discarding a result that already exists. + for _ in 0..<12 { + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: "await new Promise(resolve => setTimeout(resolve, 40)); return 'settled';", + allowedCapabilities: [], + timeoutMs: 40 + ) + ) + // Either outcome is legitimate under the race, but a fulfilled script + // must never come back empty *and* successful. + if observed.error == nil { + #expect(observed.result?.output == .string("settled")) + } else { + #expect(observed.error?.code == "EXECUTION_TIMEOUT") + } + } +} From f7db6c7b98967a6111a930f290bc816b3d3a4468 Mon Sep 17 00:00:00 2001 From: Zac White Date: Sat, 25 Jul 2026 21:19:31 -0700 Subject: [PATCH 16/18] Teach agents the paradigm, not just the API surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tool descriptions and the generated TypeScript answer "what is this helper called and what does it take". Nothing answered "what shape should my work take", so an agent could reasonably use executeJavaScript as a thin RPC — one call per operation — which is exactly the pattern this package exists to replace. Every code example in the package was a *search* example; executeJavaScript had none. `CodeModeAgentGuidance` supplies the missing half, to pair with `typeDeclarations()`: one call runs the whole task, filter and aggregate in the script, return the graded answer rather than the raw data, request only the capabilities you call. Three tiers with an optional budget selector, since prompt space is the scarce resource: - `.brief` (~160 tokens) — the core idea alone - `.standard` (~580) — adds the search → write → repair workflow and a worked multi-step example - `.full` (~1000) — adds partial-failure handling, the sequential-native-call reality so Promise.all is not over-promised, and the anti-pattern list `systemPrompt(approximateTokenBudget:)` picks the largest tier that fits. Tiers are additive — a smaller one loses examples, never a rule, which a test pins by substring containment. Token counts are estimated at four characters per token and documented as a planning bound, not a real tokenization. A budget too small for any tier still returns `.brief` rather than nothing. Also in executeJavaScript's description: the paradigm now leads instead of the namespace tour, three worked examples replace having none, and capability minimization is stated plainly — the eval suite asserts on it 38 times but the description mentioned it once in passing, so we were grading an unstated rule. Guarded, because documentation that names a helper the runtime lacks teaches the exact failure it is trying to prevent: tests check every helper and argument in both the guidance and the tool description against the live catalog, and run the worked example end to end. That caught a wrong example on the way in — `contacts.read` takes `identifiers` (a plural array), so the per-item loop it showed was the anti-pattern, not the pattern; it is now the example for "prefer one call that takes a collection". The new `fs.whole-job-one-script` eval scenario grades the standard rather than just asserting it: unit tests confirm a transcript splitting one job across three executions fails on tool order, an over-broad capability list fails even with the right answer, and the single-script transcript passes. --- README.md | 39 ++++ .../CodeMode/Host/CodeModeAgentGuidance.swift | 194 ++++++++++++++++++ .../Host/CodeModeAgentToolDescriptions.swift | 31 ++- .../CodeModeEvaluation/EvalScenarios.swift | 45 ++++ .../CodeModeEvalRunnerTests.swift | 89 ++++++++ Tests/CodeModeTests/AgentGuidanceTests.swift | 157 ++++++++++++++ 6 files changed, 553 insertions(+), 2 deletions(-) create mode 100644 Sources/CodeMode/Host/CodeModeAgentGuidance.swift create mode 100644 Tests/CodeModeTests/AgentGuidanceTests.swift diff --git a/README.md b/README.md index 210228c..261a36c 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,45 @@ let tools = CodeModeAgentTools( ) ``` +## Agent Guidance + +The tool descriptions and the generated TypeScript tell a model *what the helpers +are called*. Neither tells it *what shape its work should take* — so an agent can +end up using `executeJavaScript` as a thin RPC, one call per operation, which is +the pattern this package exists to replace. + +`CodeModeAgentGuidance` is the missing half. Pair it with `typeDeclarations()`: + +```swift +let systemPrompt = CodeModeAgentGuidance.systemPrompt(.standard) + + "\n\n" + + tools.typeDeclarations() +``` + +Three tiers, each self-contained and additive — dropping to a smaller one loses +examples, never a rule: + +| Tier | ~Tokens | Contents | +| --- | --- | --- | +| `.brief` | ~160 | The core idea: one call runs the whole task; request only the capabilities you use. | +| `.standard` | ~580 | Adds the search → write → repair workflow and a worked multi-step example. | +| `.full` | ~1000 | Adds partial-failure handling, the sequential-native-call reality, and the anti-pattern list. | + +Or let a budget choose: + +```swift +CodeModeAgentGuidance.systemPrompt(approximateTokenBudget: 400) // -> .standard +``` + +Token counts are estimated at four characters per token — a planning bound, not a +real tokenization. A budget too small for any tier still returns `.brief`, on the +grounds that some guidance is the difference between an agent that batches its +work and one that does not. + +The `execution.*` and `fs.whole-job-one-script` eval scenarios grade this: a +transcript that splits one obvious job across several executions, or that asks +for capabilities it never calls, fails. + ## TypeScript Declarations The registry already knows argument names, types, optionality, enum constraints, diff --git a/Sources/CodeMode/Host/CodeModeAgentGuidance.swift b/Sources/CodeMode/Host/CodeModeAgentGuidance.swift new file mode 100644 index 0000000..af596b2 --- /dev/null +++ b/Sources/CodeMode/Host/CodeModeAgentGuidance.swift @@ -0,0 +1,194 @@ +import Foundation + +/// System-prompt guidance that teaches an agent *why* it is writing a program +/// rather than calling tools one at a time. +/// +/// The tool descriptions and the generated TypeScript answer "what is this helper +/// called and what does it take". Neither answers "what shape should my work +/// take", and an agent that never learns the second question will use +/// `executeJavaScript` as a thin RPC — one call per operation — which is the +/// pattern this package exists to replace. +/// +/// Pair it with `CodeModeAgentTools.typeDeclarations()`, which supplies the +/// vocabulary this supplies the strategy for: +/// +/// ```swift +/// let prompt = CodeModeAgentGuidance.systemPrompt(.standard) +/// + "\n\n" +/// + tools.typeDeclarations() +/// ``` +public enum CodeModeAgentGuidance { + /// How much prompt budget to spend on the guidance. + /// + /// Every tier is self-contained and states the core idea; the larger ones add + /// worked examples and failure modes rather than qualifying what came before, + /// so a host can drop down a tier without losing a rule. + public enum Length: String, Sendable, CaseIterable, Comparable { + /// One paragraph: the core idea and nothing else. + case brief + /// The idea, the workflow, and one worked example. The default. + case standard + /// Adds fan-out and error-handling examples plus the anti-pattern list. + /// Worth it when the agent will do open-ended multi-step work. + case full + + public static func < (lhs: Length, rhs: Length) -> Bool { + lhs.order < rhs.order + } + + fileprivate var order: Int { + switch self { + case .brief: return 0 + case .standard: return 1 + case .full: return 2 + } + } + + /// Rough size of this tier's text, for budgeting against a prompt. + /// + /// Estimated at four characters per token — the usual English-prose rule + /// of thumb, not a real tokenization. Treat it as a bound to plan with, + /// not a number to assert on. + public var approximateTokenCount: Int { + CodeModeAgentGuidance.systemPrompt(self).count / 4 + } + } + + /// The guidance text at the requested length. + public static func systemPrompt(_ length: Length = .standard) -> String { + switch length { + case .brief: + return core + case .standard: + return [core, workflow, workedExample].joined(separator: "\n\n") + case .full: + return [core, workflow, workedExample, fanOutExample, antiPatterns].joined(separator: "\n\n") + } + } + + /// The largest tier that fits the budget, or `.brief` when nothing does. + /// + /// `.brief` is returned even when it exceeds the budget: some guidance is the + /// difference between an agent that batches and one that does not, so the + /// floor is deliberate rather than an empty string. + public static func systemPrompt(approximateTokenBudget: Int) -> String { + systemPrompt(length(forApproximateTokenBudget: approximateTokenBudget)) + } + + /// The tier `systemPrompt(approximateTokenBudget:)` would choose. + public static func length(forApproximateTokenBudget budget: Int) -> Length { + Length.allCases + .sorted(by: >) + .first { $0.approximateTokenCount <= budget } + ?? .brief + } + + // MARK: - Sections + + private static let core = """ + ## Writing code instead of calling tools + + You have a JavaScript runtime with direct access to this device's APIs. Use it \ + to run whole tasks, not single operations. One `executeJavaScript` call should \ + do the entire job — read, loop, branch, filter, transform, write — and return \ + only the finished answer. Ten helper calls inside one script is normal and \ + fast; ten `executeJavaScript` calls that each make one helper call is the \ + mistake this runtime exists to avoid, because every round trip costs a model \ + turn and pushes intermediate data you do not need through your context. \ + Request only the capabilities your script actually calls. + """ + + private static let workflow = """ + ## How to work + + 1. Call `searchJavaScriptAPI` first when you are unsure of helper names, \ + arguments, or result shapes. Return `ref.dts` — the TypeScript declaration — \ + rather than assembling the metadata fields yourself. + 2. Write one script that completes the task. Reach for ordinary control flow: \ + loops over collections, `if` for branching, `try`/`catch` around anything that \ + may fail per item. + 3. Do the filtering and aggregation *in the script*. Return the graded answer — \ + a count, a summary, the three matching records — not the raw collection you \ + read. A large return value is truncated, and the data you did not need cost \ + you context on the way through. + 4. Repair from the structured error. It names the capability, the line and \ + column, and what to change; the suggestions tell you when a retry cannot help. + """ + + private static let workedExample = """ + ## The shape to aim for + + Task: "how much did I spend on the trip, and file the receipts?" + + ```javascript + // ONE call. Reads, filters, sums, writes, and returns just the total. + const entries = await apple.fs.list({ path: 'documents:receipts' }); + let total = 0; + const filed = []; + + for (const entry of entries) { + if (entry.isDirectory || !entry.name.endsWith('.json')) continue; + const { text } = await apple.fs.read({ path: entry.path }); + const receipt = JSON.parse(text); + if (receipt.trip !== 'lisbon') continue; + total += receipt.amount; + await apple.fs.move({ + from: entry.path, + to: `documents:receipts/lisbon/${entry.name}`, + overwrite: true + }); + filed.push(entry.name); + } + + return { total, filedCount: filed.length }; + ``` + + `allowedCapabilities: ["fs.list", "fs.read", "fs.move"]` — exactly the three \ + helpers the script calls, and nothing else. + """ + + private static let fanOutExample = """ + ## Handling many items and partial failure + + Keep going when one item fails, and report what happened rather than throwing \ + the whole run away: + + ```javascript + const results = []; + for (const city of ['lisbon', 'porto', 'faro']) { + try { + const response = await fetch(`https://api.example.com/weather/${city}`); + if (!response.ok) { results.push({ city, error: response.status }); continue; } + const data = await response.json(); + results.push({ city, tempC: data.current.temp_c }); + } catch (error) { + results.push({ city, error: String(error) }); + } + } + return results; + ``` + + Native calls run one at a time, so `Promise.all` does not make these overlap — \ + a loop is just as fast here and reads better. Use \ + `await new Promise(r => setTimeout(r, 1000))` to back off between retries; \ + delays are real. + """ + + private static let antiPatterns = """ + ## Anti-patterns + + - **One call per operation.** If your second `executeJavaScript` starts from \ + data the first one returned, both should have been one script. + - **Returning the raw collection.** Reading 500 records to answer "how many \ + are overdue" should return a number, not 500 records. + - **Asking for capabilities you do not call.** Request the exact set your \ + script uses; a wider list is refused by hosts that enforce a ceiling, and it \ + is the wrong thing to ask a user to approve. + - **Forgetting `await`.** An un-awaited helper's failure is silently \ + discarded and you will be told the run succeeded. A \ + `BRIDGE_FAILURES_NOT_SURFACED` diagnostic means exactly this happened. + - **Re-running a call the error told you not to retry.** \ + `NETWORK_POLICY_VIOLATION`, `UI_PRESENTER_UNAVAILABLE`, and a host-withheld \ + `CAPABILITY_DENIED` will not succeed on a second attempt. + """ +} diff --git a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift index 2049743..7585c47 100644 --- a/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift +++ b/Sources/CodeMode/Host/CodeModeAgentToolDescriptions.swift @@ -124,15 +124,42 @@ public enum CodeModeAgentToolDescriptions { public static let executeJavaScript = CodeModeAgentToolDescription( name: "executeJavaScript", description: """ - Execute JavaScript against the CodeMode runtime. Prefer searchJavaScriptAPI first when choosing helpers or arguments. Cross-platform helpers live under apple.* and platform-specific helpers live under platform namespaces such as ios.alarm.*; custom host providers may expose additional namespaces. System UI helpers such as apple.ui.presentAlert, apple.calendar.presentNewEvent, apple.photos.pick, apple.contacts.pick, apple.documents.pick, apple.share.present, apple.quicklook.preview, apple.web.present, and apple.auth.webAuthenticate require a host-provided SystemUIPresenter; camera, document scan, mail, and message compose helpers are iOS-only. Only helpers supported on the current host platform are installed. + Execute JavaScript against the CodeMode runtime. Use one call to run a whole task — read, loop, branch, filter, transform, write — and return only the finished answer. Many helper calls inside one script is the intended shape; many executeJavaScript calls that each make one helper call is not, because every round trip costs a turn and pushes intermediate data through your context. Prefer searchJavaScriptAPI first when choosing helpers or arguments. Cross-platform helpers live under apple.* and platform-specific helpers live under platform namespaces such as ios.alarm.*; custom host providers may expose additional namespaces. System UI helpers such as apple.ui.presentAlert, apple.calendar.presentNewEvent, apple.photos.pick, apple.contacts.pick, apple.documents.pick, apple.share.present, apple.quicklook.preview, apple.web.present, and apple.auth.webAuthenticate require a host-provided SystemUIPresenter; camera, document scan, mail, and message compose helpers are iOS-only. Only helpers supported on the current host platform are installed. Calling conventions: apple.* and other namespaced helpers take one object argument, for example apple.fs.read({ path: "tmp:file.txt" }) and often return structured objects such as { text, base64 }. Node-style filesystem aliases under fs.promises.* use positional arguments, for example fs.promises.readFile("tmp:file.txt", "utf8"), and return Node-like values such as a string for readFile. When in doubt, use the ref.dts from searchJavaScriptAPI — it states the exact signature. fetch(url, options) is a global helper and returns a Response-like object with text(), json(), headers.get(), status, and ok. Return semantics: the runtime wraps your code in an async function. For multi-statement code, return the final graded value with an explicit top-level return statement. A script that is only a bare final top-level await expression, such as await apple.fs.read({ path: "tmp:file.txt" }), returns that awaited value. Do not use an unreturned async IIFE as the final expression. + Examples. Do the whole job in one script and return the graded answer, not the raw data you read: + + const start = new Date().toISOString(); + const end = new Date(Date.now() + 7 * 86400000).toISOString(); + const events = await apple.calendar.listEvents({ start, end }); + const conflicts = events.filter((event, index) => + events.slice(index + 1).some(other => other.startDate < event.endDate) + ); + return { total: events.length, conflicts: conflicts.map(event => event.title) }; + + Prefer one call that takes a collection over a loop of single calls — check the catalog for a plural argument before writing the loop: + + const contacts = await apple.contacts.list({ identifiers: ids }); + + When the work genuinely is per-item, keep going after a failure and report what happened: + + const results = []; + for (const city of cities) { + try { + const response = await fetch(`https://api.example.com/weather/${city}`); + results.push({ city, ok: response.ok, data: response.ok ? await response.json() : null }); + } catch (error) { + results.push({ city, error: String(error) }); + } + } + return results; + Concurrency: setTimeout and clearTimeout work normally — callbacks are deferred and delays are honoured, so `await new Promise(r => setTimeout(r, 1000))` really waits. There is no I/O event loop, though: native calls run one at a time, so Promise.all over several helpers completes them sequentially rather than concurrently. A promise with no resolve path and no pending timer can never settle and fails fast with JS_RUNTIME_ERROR rather than running out the clock. - Allowlisting: include only the required built-in capabilities in allowedCapabilities and only custom provider keys in allowedCapabilityKeys. The two fields are not interchangeable: a built-in capability listed in allowedCapabilityKeys is ignored. These fields declare what your script needs; the host app applies its own ceiling on top, so a capability you list may still be withheld. Execution defaults to a 10000ms timeout and returns structured CodeModeToolError failures for syntax errors, missing JS helpers, runtime throws, validation failures, permission denials, timeouts, cancellation, and internal errors. + Allowlisting: request the exact set of capabilities your script calls and nothing more — a capability you list but never use is a wider grant than the task needs, and hosts that enforce a ceiling will refuse it. Built-in capability IDs go in allowedCapabilities; custom provider keys go in allowedCapabilityKeys. The two fields are not interchangeable: a built-in capability listed in allowedCapabilityKeys is ignored. These fields declare what your script needs; the host app applies its own ceiling on top, so a capability you list may still be withheld. Execution defaults to a 10000ms timeout and returns structured CodeModeToolError failures for syntax errors, missing JS helpers, runtime throws, validation failures, permission denials, timeouts, cancellation, and internal errors. Error repair guide: JS_API_NOT_FOUND: use the suggested JS helper names or searchJavaScriptAPI. diff --git a/Sources/CodeModeEvaluation/EvalScenarios.swift b/Sources/CodeModeEvaluation/EvalScenarios.swift index da0a636..44359cb 100644 --- a/Sources/CodeModeEvaluation/EvalScenarios.swift +++ b/Sources/CodeModeEvaluation/EvalScenarios.swift @@ -18,6 +18,7 @@ public enum CodeModeEvalScenarios { executionTimeout, executionUnsettleablePromise, executionTimerBackoff, + filesystemWholeJobInOneScript, reminderCatalogDiscovery, catalogFileSystemReadShape, catalogConsoleDiagnostics, @@ -528,6 +529,50 @@ public enum CodeModeEvalScenarios { ) ) + public static let filesystemWholeJobInOneScript = CodeModeEvalScenario( + id: "fs.whole-job-one-script", + title: "A multi-step job runs as one script", + task: "Read every .json receipt in documents:receipts, total the amounts for trip 'lisbon', and return { total, count }. Do the whole job in a single executeJavaScript call — list, read, filter, and sum inside the script — and return only the totals, not the receipts.", + searchCode: """ + async () => { + return api.references + .filter(ref => ["fs.list", "fs.read"].includes(ref.capability)) + .map(ref => ref.dts) + .join("\\n\\n"); + } + """, + executeCode: """ + const entries = await apple.fs.list({ path: 'documents:receipts' }); + let total = 0; + let count = 0; + for (const entry of entries) { + if (entry.isDirectory || !entry.name.endsWith('.json')) continue; + const { text } = await apple.fs.read({ path: entry.path }); + const receipt = JSON.parse(text); + if (receipt.trip !== 'lisbon') continue; + total += receipt.amount; + count += 1; + } + return { total, count }; + """, + allowedCapabilities: [.fsList, .fsRead], + seedFiles: [ + CodeModeEvalSeedFile(path: "documents:receipts/a.json", text: #"{"trip":"lisbon","amount":12}"#), + CodeModeEvalSeedFile(path: "documents:receipts/b.json", text: #"{"trip":"porto","amount":99}"#), + CodeModeEvalSeedFile(path: "documents:receipts/c.json", text: #"{"trip":"lisbon","amount":30}"#), + ], + expectation: CodeModeEvalExpectation( + // One search, one execute. A transcript that splits the list/read/sum + // across several executions fails here — that is the whole point of + // the scenario, and it only has teeth against real LLM transcripts. + toolOrder: [.searchJavaScriptAPI, .executeJavaScript], + exactAllowedCapabilities: [.fsList, .fsRead], + forbiddenCapabilities: [.fsWrite, .fsDelete, .fsMove], + requiredExecuteCodeFragments: ["apple.fs.list", "apple.fs.read"], + expectedOutput: .object(["total": .number(42), "count": .number(2)]) + ) + ) + public static let executionTimerBackoff = CodeModeEvalScenario( id: "execution.timer-backoff", title: "setTimeout honours its delay", diff --git a/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift b/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift index 269749c..5580d09 100644 --- a/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift +++ b/Tests/CodeModeEvalTests/CodeModeEvalRunnerTests.swift @@ -197,3 +197,92 @@ private func failureSummary(_ results: [CodeModeEvalResult]) -> String { #expect(result.toolCalls.filter { $0.tool == .executeJavaScript }.count == 1) #expect(result.executionOutput != .string("second step succeeded")) } + +// MARK: - Grading the one-script-per-job standard + +// The docs now tell agents to do a whole job in one executeJavaScript call. That +// is only a real standard if a transcript that ignores it fails, so this grades +// the shape rather than just asserting the reference solution. + +@Test func aTranscriptSplitAcrossExecutionsFailsTheOneScriptScenario() { + let scenario = CodeModeEvalScenarios.filesystemWholeJobInOneScript + + // The anti-pattern: list in one call, read in another, sum in a third — + // every intermediate result round-tripped through the model's context. + let split = [ + CodeModeEvalToolCall(tool: .searchJavaScriptAPI, code: scenario.searchCode ?? ""), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: "return await apple.fs.list({ path: 'documents:receipts' });", + allowedCapabilities: [.fsList] + ), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: "return await apple.fs.read({ path: 'documents:receipts/a.json' });", + allowedCapabilities: [.fsRead] + ), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: "return await apple.fs.read({ path: 'documents:receipts/c.json' });", + allowedCapabilities: [.fsRead] + ), + ] + + let failures = CodeModeEvalRunner().validateTranscript( + scenario: scenario, + toolCalls: split, + searchResult: .string("fs.list fs.read"), + executionOutput: .object(["total": .number(42), "count": .number(2)]), + error: nil + ) + + #expect(failures.contains { $0.contains("Tool order") }) +} + +@Test func theSingleScriptTranscriptPassesTheOneScriptScenario() { + let scenario = CodeModeEvalScenarios.filesystemWholeJobInOneScript + + let single = [ + CodeModeEvalToolCall(tool: .searchJavaScriptAPI, code: scenario.searchCode ?? ""), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: scenario.executeCode ?? "", + allowedCapabilities: [.fsList, .fsRead] + ), + ] + + let failures = CodeModeEvalRunner().validateTranscript( + scenario: scenario, + toolCalls: single, + searchResult: .string("fs.list fs.read"), + executionOutput: .object(["total": .number(42), "count": .number(2)]), + error: nil + ) + + #expect(failures.isEmpty, "\(failures)") +} + +@Test func anOverBroadCapabilityRequestFailsEvenWithTheRightAnswer() { + let scenario = CodeModeEvalScenarios.filesystemWholeJobInOneScript + + // Minimization is now stated plainly in the tool description, so grading it + // is no longer an unstated rule. + let overBroad = [ + CodeModeEvalToolCall(tool: .searchJavaScriptAPI, code: scenario.searchCode ?? ""), + CodeModeEvalToolCall( + tool: .executeJavaScript, + code: scenario.executeCode ?? "", + allowedCapabilities: [.fsList, .fsRead, .fsWrite] + ), + ] + + let failures = CodeModeEvalRunner().validateTranscript( + scenario: scenario, + toolCalls: overBroad, + searchResult: .string("fs.list fs.read"), + executionOutput: .object(["total": .number(42), "count": .number(2)]), + error: nil + ) + + #expect(failures.contains { $0.contains("Allowed capabilities") || $0.contains("Forbidden capabilities") }) +} diff --git a/Tests/CodeModeTests/AgentGuidanceTests.swift b/Tests/CodeModeTests/AgentGuidanceTests.swift new file mode 100644 index 0000000..d3bf642 --- /dev/null +++ b/Tests/CodeModeTests/AgentGuidanceTests.swift @@ -0,0 +1,157 @@ +import Foundation +import Testing +@testable import CodeMode + +// The tool descriptions and the generated TypeScript teach an agent what the +// helpers are called. Nothing taught it what shape its work should take, so an +// agent could reasonably use executeJavaScript as a thin RPC — one call per +// operation — which is the pattern this package exists to replace. + +@Test func everyTierStatesTheCoreIdea() { + // A host dropping to a smaller tier must not lose a rule, only the examples. + for length in CodeModeAgentGuidance.Length.allCases { + let text = CodeModeAgentGuidance.systemPrompt(length) + #expect(text.contains("One `executeJavaScript` call should do the entire job")) + #expect(text.contains("Request only the capabilities your script actually calls")) + } +} + +@Test func tiersGrowStrictlyAndAreAdditive() { + let brief = CodeModeAgentGuidance.systemPrompt(.brief) + let standard = CodeModeAgentGuidance.systemPrompt(.standard) + let full = CodeModeAgentGuidance.systemPrompt(.full) + + #expect(brief.count < standard.count) + #expect(standard.count < full.count) + // Additive, not rewritten: the larger tiers contain the smaller ones verbatim. + #expect(standard.contains(brief)) + #expect(full.contains(standard)) +} + +@Test func largerTiersAddExamplesAndFailureModes() { + let standard = CodeModeAgentGuidance.systemPrompt(.standard) + let full = CodeModeAgentGuidance.systemPrompt(.full) + + #expect(standard.contains("```javascript")) + #expect(standard.contains("searchJavaScriptAPI")) + #expect(full.contains("Anti-patterns")) + #expect(full.contains("BRIDGE_FAILURES_NOT_SURFACED")) + // The concurrency reality must not be over-promised in the fan-out example. + #expect(full.contains("`Promise.all` does not make these overlap")) +} + +@Test func aBudgetSelectsTheLargestTierThatFits() { + let brief = CodeModeAgentGuidance.Length.brief.approximateTokenCount + let standard = CodeModeAgentGuidance.Length.standard.approximateTokenCount + let full = CodeModeAgentGuidance.Length.full.approximateTokenCount + + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: full) == .full) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: full + 5_000) == .full) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: full - 1) == .standard) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: standard) == .standard) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: standard - 1) == .brief) + #expect(CodeModeAgentGuidance.length(forApproximateTokenBudget: brief) == .brief) +} + +@Test func aBudgetTooSmallForAnythingStillReturnsTheCoreIdea() { + // Deliberately a floor rather than an empty string: some guidance is the + // difference between an agent that batches and one that does not. + #expect(CodeModeAgentGuidance.systemPrompt(approximateTokenBudget: 0) == CodeModeAgentGuidance.systemPrompt(.brief)) + #expect(CodeModeAgentGuidance.systemPrompt(approximateTokenBudget: -100) == CodeModeAgentGuidance.systemPrompt(.brief)) +} + +@Test func guidanceExamplesUseHelpersAndCapabilitiesThatActuallyExist() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let full = CodeModeAgentGuidance.systemPrompt(.full) + let jsNames = Set(tools.capabilities().flatMap(\.jsNames)) + + // Guidance naming a helper the runtime does not install would teach the exact + // failure it is trying to prevent. + for helper in ["apple.fs.list", "apple.fs.read", "apple.fs.move"] { + #expect(full.contains(helper)) + #expect(jsNames.contains(helper), "guidance references missing helper \(helper)") + } + + for capability in ["fs.list", "fs.read", "fs.move"] { + #expect(CapabilityID(rawValue: capability) != nil, "guidance references missing capability \(capability)") + } +} + +@Test func theWorkedExampleActuallyRuns() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // The example is the shape we ask agents to copy, so it has to be a script + // this runtime accepts — arguments, calling conventions, and all. The setup + // lines are the only addition. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + await apple.fs.mkdir({ path: 'documents:receipts/lisbon' }); + await apple.fs.write({ + path: 'documents:receipts/a.json', + data: JSON.stringify({ trip: 'lisbon', amount: 12 }) + }); + await apple.fs.write({ + path: 'documents:receipts/b.json', + data: JSON.stringify({ trip: 'porto', amount: 99 }) + }); + + const entries = await apple.fs.list({ path: 'documents:receipts' }); + let total = 0; + const filed = []; + + for (const entry of entries) { + if (entry.isDirectory || !entry.name.endsWith('.json')) continue; + const { text } = await apple.fs.read({ path: entry.path }); + const receipt = JSON.parse(text); + if (receipt.trip !== 'lisbon') continue; + total += receipt.amount; + await apple.fs.move({ + from: entry.path, + to: `documents:receipts/lisbon/${entry.name}`, + overwrite: true + }); + filed.push(entry.name); + } + + return { total, filedCount: filed.length }; + """, + allowedCapabilities: [.fsMkdir, .fsWrite, .fsList, .fsRead, .fsMove] + ) + ) + + #expect(observed.error == nil) + let output = try #require(observed.result?.output?.objectValue) + #expect(output.int("total") == 12) + #expect(output.int("filedCount") == 1) +} + +@Test func toolDescriptionExamplesNameRealHelpersAndArguments() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + let description = CodeModeAgentToolDescriptions.executeJavaScript.description + let references = tools.capabilities() + let byJSName = Dictionary(references.flatMap { ref in ref.jsNames.map { ($0, ref) } }, uniquingKeysWith: { first, _ in first }) + + // An example that names a helper or argument the runtime does not have + // teaches the exact failure the examples exist to prevent — and a wrong + // argument name is the easiest kind to get wrong when writing prose. + let used: [(helper: String, arguments: [String])] = [ + ("apple.calendar.listEvents", ["start", "end"]), + ("apple.contacts.list", ["identifiers"]), + ] + + for (helper, arguments) in used { + #expect(description.contains(helper)) + let reference = try #require(byJSName[helper], "description references missing helper \(helper)") + let known = Set(reference.requiredArguments + reference.optionalArguments) + for argument in arguments { + #expect(known.contains(argument), "\(helper) has no argument '\(argument)'") + } + } +} From ed42fa2b767674a58720df5360e3190910674edb Mon Sep 17 00:00:00 2001 From: Zac White Date: Mon, 27 Jul 2026 15:15:27 -0700 Subject: [PATCH 17/18] Fix CI: iOS/visionOS build breaks and load-sensitive test bounds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four failures on PR #7, all reproducible locally with the exact CI commands. **iOS build — a real break in my own CoreLocation fix.** `LocationPermissionDelegate` crosses an isolation boundary: the manager is created inside a `DispatchQueue.main.async` block, which is main-actor-isolated on iOS, while `wait`/`observedStatus` are called from the requesting thread. Swift 6 rejects sending a non-Sendable delegate across that. It is genuinely thread-safe — all mutable state is lock-guarded, the handoff is a semaphore — so it is now `@unchecked Sendable` with the reasoning recorded. `swift test` on macOS never caught this because `requestLocationPermission` is inside `#if canImport(CoreLocation) && os(iOS)`, so the macOS build never compiled the branch. Exactly the gap TODO.md flags about iOS being build-only. **visionOS build — latent, surfaced by pinning the runner.** `openSystemURL` calls main-actor-isolated `UIApplication.shared.open` from a nonisolated synchronous context. This is pre-existing code I did not touch; pinning `macos-latest` to `macos-15` moved the toolchain to one that diagnoses it. Fixed with `MainActor.assumeIsolated` on the branch that has already established it is on the main thread — a hop would return before the open was requested. **Three load-sensitive test bounds.** Individual tests took 8-9s on the CI runner against ~0.1s locally, so wall-clock upper bounds written on an idle machine failed. Each of these is a hang-guard, not a performance assertion, and the thresholds now say so: - `setTimeoutActuallyWaitsTheRequestedDelay` keeps its lower bound, which is the actual claim (the delay was previously ignored entirely); the upper bound is dropped since the script's own 5s timeoutMs already fails a runaway wait. - The remaining bounds are loosened to stay well under the thing they prove did not happen — 20s against a 30s budget, 15s against a 20s timer. - `completionWaitToleratesALateCallback` waits 60s. The starvation is real: parallel tests plus this runtime's thread-blocking timer waits can leave a 0.2s dispatch waiting seconds for a GCD worker. That is a property of the synchronous bridge model, and one more argument for the async ABI in #9. Also fixes the `variable 'root' was never mutated` warning CI surfaced in TypeScriptDeclarations. Verified locally with the exact CI invocations: `swift test` (322 passing) plus `xcodebuild -destination generic/platform={iOS,visionOS}`, both succeeding. --- .../API/SystemAppleServiceClients.swift | 10 ++++++++-- .../Catalog/TypeScriptDeclarations.swift | 2 +- .../Security/SystemPermissionBroker.swift | 7 ++++++- Tests/CodeModeTests/CompletionWaitTests.swift | 5 ++++- Tests/CodeModeTests/EventLoopTests.swift | 18 +++++++++++++----- .../CodeModeTests/ExecutionWatchdogTests.swift | 11 ++++++++--- 6 files changed, 40 insertions(+), 13 deletions(-) diff --git a/Sources/CodeMode/API/SystemAppleServiceClients.swift b/Sources/CodeMode/API/SystemAppleServiceClients.swift index 1f3db40..c501be0 100644 --- a/Sources/CodeMode/API/SystemAppleServiceClients.swift +++ b/Sources/CodeMode/API/SystemAppleServiceClients.swift @@ -819,8 +819,14 @@ enum SystemMapsMapping { private func openSystemURL(_ url: URL) -> Bool { #if canImport(UIKit) if Thread.isMainThread { - UIApplication.shared.open(url, options: [:], completionHandler: nil) - return true + // `UIApplication.shared` and `open` are main-actor-isolated. We have + // already established we are on the main thread, so state that to the + // compiler rather than hopping — a hop here would return before the open + // has been requested. + return MainActor.assumeIsolated { + UIApplication.shared.open(url, options: [:], completionHandler: nil) + return true + } } let semaphore = DispatchSemaphore(value: 0) let opened = LockedBox(false) diff --git a/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift b/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift index 6edd257..f3a2731 100644 --- a/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift +++ b/Sources/CodeMode/Catalog/TypeScriptDeclarations.swift @@ -98,7 +98,7 @@ public enum TypeScriptDeclarations { /// The whole surface: the preamble plus every capability, grouped into /// namespaces by its canonical dotted JavaScript name. public static func surface(for references: [JavaScriptAPIReference]) -> String { - var root = Namespace(name: "") + let root = Namespace(name: "") for reference in references.sorted(by: { $0.capability < $1.capability }) { guard let canonical = canonicalName(for: reference) else { continue diff --git a/Sources/CodeMode/Security/SystemPermissionBroker.swift b/Sources/CodeMode/Security/SystemPermissionBroker.swift index 36f4510..27fd3f1 100644 --- a/Sources/CodeMode/Security/SystemPermissionBroker.swift +++ b/Sources/CodeMode/Security/SystemPermissionBroker.swift @@ -711,7 +711,12 @@ public final class SystemPermissionBroker: PermissionBroker, @unchecked Sendable } #if canImport(CoreLocation) && os(iOS) -private final class LocationPermissionDelegate: NSObject, CLLocationManagerDelegate { +/// `@unchecked Sendable` because it is genuinely thread-safe and has to cross an +/// isolation boundary: the manager is created inside a `DispatchQueue.main.async` +/// block — main-actor-isolated on iOS — while `wait`/`observedStatus` are called +/// from the requesting thread. All mutable state is lock-guarded and the handoff +/// is a semaphore. +private final class LocationPermissionDelegate: NSObject, CLLocationManagerDelegate, @unchecked Sendable { private let semaphore = DispatchSemaphore(value: 0) private let lock = NSLock() private var status: CLAuthorizationStatus? diff --git a/Tests/CodeModeTests/CompletionWaitTests.swift b/Tests/CodeModeTests/CompletionWaitTests.swift index 12df116..d72171e 100644 --- a/Tests/CodeModeTests/CompletionWaitTests.swift +++ b/Tests/CodeModeTests/CompletionWaitTests.swift @@ -40,7 +40,10 @@ import Testing #expect(requireBridgeErrorCode(error) == "EXECUTION_TIMEOUT") } - #expect(lateCallbackFinished.wait(timeout: .now() + 5) == .success) + // Generous on purpose: the suite runs in parallel and this runtime's timer + // waits block GCD workers, so a 0.2s dispatch can be starved for seconds on a + // loaded runner. The assertion is that the late callback lands at all. + #expect(lateCallbackFinished.wait(timeout: .now() + 60) == .success) } @Test func completionWaitReportsWhetherTheCallbackArrived() { diff --git a/Tests/CodeModeTests/EventLoopTests.swift b/Tests/CodeModeTests/EventLoopTests.swift index 25982b3..8de2569 100644 --- a/Tests/CodeModeTests/EventLoopTests.swift +++ b/Tests/CodeModeTests/EventLoopTests.swift @@ -47,9 +47,11 @@ import Testing let elapsed = Date().timeIntervalSince(started) #expect(observed.result?.output == .string("done")) - // The delay was previously ignored entirely, so a backoff loop spun. + // The assertion is the lower bound: the delay was previously ignored + // entirely, so a backoff loop spun. There is deliberately no tight upper + // bound — a shared CI runner can stretch a 150ms wait by seconds, and the + // script's own 5s timeoutMs already fails the test if the wait runs away. #expect(elapsed >= 0.14) - #expect(elapsed < 3) } @Test func clearTimeoutCancelsAPendingCallback() async throws { @@ -135,7 +137,9 @@ import Testing #expect(observed.error?.code == "JS_RUNTIME_ERROR") #expect(observed.error?.message.contains("can never settle") == true) - #expect(Date().timeIntervalSince(started) < 5) + // Proves the 30s budget was not waited out. Loose against runner load; the + // margin against 30s is what makes it meaningful. + #expect(Date().timeIntervalSince(started) < 20) } @Test func aTimerStillPendingAtTheDeadlineTimesOut() async throws { @@ -154,7 +158,10 @@ import Testing ) #expect(observed.error?.code == "EXECUTION_TIMEOUT") - #expect(Date().timeIntervalSince(started) < 5) + // The claim is that the 30s timer was not waited out, not that the run was + // fast. Loose enough to survive a loaded runner, tight enough to still catch + // the regression it exists for. + #expect(Date().timeIntervalSince(started) < 20) } @Test func aLongBackoffStaysCancellable() async throws { @@ -177,7 +184,8 @@ import Testing let started = Date() let observed = await observe(call) #expect(observed.error?.code == "CANCELLED") - #expect(Date().timeIntervalSince(started) < 10) + // Proves cancel interrupted the 20s timer rather than waiting it out. + #expect(Date().timeIntervalSince(started) < 15) } // MARK: - Swallowed bridge failures diff --git a/Tests/CodeModeTests/ExecutionWatchdogTests.swift b/Tests/CodeModeTests/ExecutionWatchdogTests.swift index f562599..3c32f14 100644 --- a/Tests/CodeModeTests/ExecutionWatchdogTests.swift +++ b/Tests/CodeModeTests/ExecutionWatchdogTests.swift @@ -18,7 +18,10 @@ import Testing #expect(observed.result == nil) #expect(observed.error?.code == "EXECUTION_TIMEOUT") - #expect(Date().timeIntervalSince(started) < 5) + // A hang-guard, not a performance assertion: `while (true) {}` must be + // terminated at all. The error code above is the real check, and the bound is + // loose because a shared CI runner is not an idle machine. + #expect(Date().timeIntervalSince(started) < 20) } @Test func executeTerminatesCPUBoundLoopInsidePromiseChain() async throws { @@ -155,7 +158,8 @@ import Testing // "must be JSON-serializable" it used to report, which pointed the model at // the wrong repair entirely. #expect(observed.error?.code == "EXECUTION_TIMEOUT") - #expect(Date().timeIntervalSince(started) < 5) + // Same hang-guard reasoning: the runaway getter must be terminated at all. + #expect(Date().timeIntervalSince(started) < 20) } @Test func serializationBudgetDoesNotScaleWithTheExecutionTimeout() async throws { @@ -176,7 +180,8 @@ import Testing ) #expect(observed.error?.code == "EXECUTION_TIMEOUT") - #expect(Date().timeIntervalSince(started) < 10) + // Proves serialization did not inherit the script's 30s budget. + #expect(Date().timeIntervalSince(started) < 20) } @Test func searchTerminatesCPUBoundInfiniteLoop() async throws { From 7600de14778fdf858354cea88dfccf2094a2a7ec Mon Sep 17 00:00:00 2001 From: Zac White Date: Mon, 27 Jul 2026 15:23:06 -0700 Subject: [PATCH 18/18] Fire timers in due order, and stop two tests starving on GCD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI caught a real bug in the timer queue, not just a flake. `advanceTimers` sorted due timers by registration id alone. That is correct only when everything came due at the same instant. When the host's sleep overshoots far enough that two timers with *different* delays land in one tick — which is routine on a loaded machine — the later one fired first purely because it was registered first. Timers now sort by due time, using registration order only to break ties, which is what a real event loop does. The existing test could not catch this: with delays of 5ms and 10ms it only fails when the sleep overshoots, so it passed locally and failed on CI. It is joined by one that reproduces the condition deterministically — delays below the host's sleep granularity, so both timers reliably come due in a single call — plus one covering the same-instant tie-break the id sort did get right. Verified by reverting the fix: the new test fails, the old one still passes. Separately, `completionWaitReturnsTheDeliveredValue` timed out waiting 5s for an immediate `DispatchQueue.global().async`. That is thread-pool starvation: the suite runs in parallel and this runtime's timer waits block GCD workers with `Thread.sleep`, so a trivial dispatch can wait seconds for a thread. The timeout is now 60s, since the assertion is that the value arrives rather than how fast. Worth recording that the starvation is a property of the synchronous bridge model — the async ABI in #9 is what actually removes it. --- .../CodeMode/Runtime/RuntimeJavaScript.swift | 11 +++- Tests/CodeModeTests/CompletionWaitTests.swift | 7 ++- Tests/CodeModeTests/EventLoopTests.swift | 57 ++++++++++++++++++- 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift index 34a7ddc..03b88ba 100644 --- a/Sources/CodeMode/Runtime/RuntimeJavaScript.swift +++ b/Sources/CodeMode/Runtime/RuntimeJavaScript.swift @@ -53,8 +53,15 @@ enum RuntimeJavaScript { if (entry.dueIn <= 0) { due.push(entry); } else { remaining.push(entry); } }); timers.entries = remaining; - // Registration order breaks ties, matching a real event loop. - due.sort(function(a, b){ return a.id - b.id; }); + // Due time first, registration order only to break ties — the order a + // real event loop fires in. Sorting by id alone was wrong whenever the + // host's sleep overshot far enough for two timers with *different* + // delays to come due in the same tick: the later one fired first purely + // because it was registered first. `dueIn` is now negative for anything + // overdue, so ascending puts the most overdue first. + due.sort(function(a, b){ + return a.dueIn === b.dueIn ? a.id - b.id : a.dueIn - b.dueIn; + }); const errors = []; due.forEach(function(entry){ try { diff --git a/Tests/CodeModeTests/CompletionWaitTests.swift b/Tests/CodeModeTests/CompletionWaitTests.swift index d72171e..0c91fce 100644 --- a/Tests/CodeModeTests/CompletionWaitTests.swift +++ b/Tests/CodeModeTests/CompletionWaitTests.swift @@ -8,7 +8,12 @@ import Testing // "the API returned nothing". @Test func completionWaitReturnsTheDeliveredValue() throws { - let value = try CompletionWait.value(timeout: 5, operationName: "test") { complete in + // 60s, not 5: the suite runs in parallel and this runtime's timer waits block + // GCD workers with `Thread.sleep`, so even an immediate `global().async` can + // wait seconds for a thread on a loaded runner. The assertion is that the + // delivered value comes back, not how fast — and the thread pressure itself is + // a property of the synchronous bridge model that the async ABI (#9) removes. + let value = try CompletionWait.value(timeout: 60, operationName: "test") { complete in DispatchQueue.global().async { complete([1, 2, 3]) } } #expect(value == [1, 2, 3]) diff --git a/Tests/CodeModeTests/EventLoopTests.swift b/Tests/CodeModeTests/EventLoopTests.swift index 8de2569..3f2f0b1 100644 --- a/Tests/CodeModeTests/EventLoopTests.swift +++ b/Tests/CodeModeTests/EventLoopTests.swift @@ -75,10 +75,13 @@ import Testing #expect(observed.result?.output?.objectValue?.array("fired") == []) } -@Test func timersFireInRegistrationOrderWhenDueTogether() async throws { +@Test func timersFireInDueOrderNotRegistrationOrder() async throws { let (tools, sandbox) = try makeTools() defer { cleanup(sandbox) } + // Registered late-first, so registration order and due order disagree. This + // must hold however far the host's sleep overshoots — if enough time passes + // for both to come due in one tick, they still fire shortest-delay first. let observed = try await execute( tools, request: JavaScriptExecutionRequest( @@ -96,6 +99,58 @@ import Testing #expect(observed.result?.output?.objectValue?.array("order") == [.string("a"), .string("b")]) } +@Test func timersComingDueInOneTickStillFireInDueOrder() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Delays below the host's sleep granularity, so both timers reliably come due + // inside a *single* advanceTimers call — the case that sorting by + // registration id alone got wrong. Registered late-first so the two orderings + // disagree; this fails deterministically against the old implementation + // rather than only on a loaded runner. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => order.push('later'), 2); + setTimeout(() => order.push('sooner'), 1); + await new Promise(resolve => setTimeout(resolve, 40)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect(observed.result?.output?.objectValue?.array("order") == [.string("sooner"), .string("later")]) +} + +@Test func timersDueAtTheSameInstantFireInRegistrationOrder() async throws { + let (tools, sandbox) = try makeTools() + defer { cleanup(sandbox) } + + // Equal delays, so only registration order can break the tie. + let observed = try await execute( + tools, + request: JavaScriptExecutionRequest( + code: """ + const order = []; + setTimeout(() => order.push('first'), 5); + setTimeout(() => order.push('second'), 5); + setTimeout(() => order.push('third'), 5); + await new Promise(resolve => setTimeout(resolve, 30)); + return { order }; + """, + allowedCapabilities: [] + ) + ) + + #expect( + observed.result?.output?.objectValue?.array("order") + == [.string("first"), .string("second"), .string("third")] + ) +} + @Test func anErrorInATimerCallbackBecomesADiagnosticNotACallerThrow() async throws { let (tools, sandbox) = try makeTools() defer { cleanup(sandbox) }