From 1c8b1f0194a9f3538482dae7c3986654b13e9b20 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 03:29:35 +0000 Subject: [PATCH 1/7] Add Linux CI job and portability guards Run the test suite on ubuntu-24.04 alongside macOS. Guard the Darwin-only surfaces so the package builds on Linux: - SecurityScopedFilesystem (security scopes and URL bookmarks) is compiled only where Darwin is available. - SandboxFilesystem app-group resolution throws unsupported on platforms without container APIs. - OverlayFilesystem import classifies entries via FileManager attributes instead of URL resource values. The workflow also runs on pushes to core-improvements so changes are verified before a PR exists. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01G29SXqzHqDycqXGa4dRr4x --- .github/workflows/pr-tests.yml | 19 ++++++++++++++----- Sources/Workspace/FS/OverlayFilesystem.swift | 6 +++--- Sources/Workspace/FS/SandboxFilesystem.swift | 4 ++++ .../FS/SecurityScopedFilesystem.swift | 2 ++ Tests/WorkspaceTests/FilesystemTests.swift | 11 +++++++++++ 5 files changed, 34 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 712af8b..e7f38b1 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -7,15 +7,24 @@ on: - reopened - synchronize - ready_for_review + push: + branches: + - core-improvements concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: swift-tests: - name: Swift Tests - runs-on: macos-latest + name: Swift Tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - ubuntu-24.04 timeout-minutes: 15 permissions: @@ -49,7 +58,7 @@ jobs: if: ${{ !cancelled() }} uses: dorny/test-reporter@v3 with: - name: Swift Test Report + name: Swift Test Report (${{ matrix.os }}) path: .build/test-results/xunit.xml reporter: swift-xunit fail-on-error: false @@ -93,7 +102,7 @@ jobs: if: always() uses: actions/upload-artifact@v6 with: - name: pr-test-results-${{ github.event.pull_request.number }} + name: pr-test-results-${{ matrix.os }}-${{ github.event.pull_request.number || github.run_id }} if-no-files-found: warn include-hidden-files: true path: | diff --git a/Sources/Workspace/FS/OverlayFilesystem.swift b/Sources/Workspace/FS/OverlayFilesystem.swift index e563a18..5c38bab 100644 --- a/Sources/Workspace/FS/OverlayFilesystem.swift +++ b/Sources/Workspace/FS/OverlayFilesystem.swift @@ -143,12 +143,12 @@ public actor OverlayFilesystem: FileSystem { } private func importItem(at url: URL, virtualPath: WorkspacePath, into overlay: InMemoryFilesystem) async throws { - let values = try url.resourceValues(forKeys: [.isDirectoryKey, .isSymbolicLinkKey]) let attributes = try fileManager.attributesOfItem(atPath: url.path) + let fileType = attributes[.type] as? FileAttributeType let permissionBits = (attributes[.posixPermissions] as? NSNumber)?.intValue let permissions = permissionBits.map(POSIXPermissions.init(_:)) - if values.isSymbolicLink == true { + if fileType == .typeSymbolicLink { let target = try fileManager.destinationOfSymbolicLink(atPath: url.path) try await overlay.createSymlink(path: virtualPath, target: target) if let permissions { @@ -157,7 +157,7 @@ public actor OverlayFilesystem: FileSystem { return } - if values.isDirectory == true { + if fileType == .typeDirectory { try await overlay.createDirectory(path: virtualPath, recursive: true) if let permissions { try await overlay.setPermissions(path: virtualPath, permissions: permissions) diff --git a/Sources/Workspace/FS/SandboxFilesystem.swift b/Sources/Workspace/FS/SandboxFilesystem.swift index 9ca8da1..2666fba 100644 --- a/Sources/Workspace/FS/SandboxFilesystem.swift +++ b/Sources/Workspace/FS/SandboxFilesystem.swift @@ -125,10 +125,14 @@ public final class SandboxFilesystem: FileSystem, Sendable { guard identifier.hasPrefix("group.") else { throw WorkspaceError.unsupported("invalid app group identifier: \(identifier)") } + #if canImport(Darwin) guard let url = fileManager.containerURL(forSecurityApplicationGroupIdentifier: identifier) else { throw WorkspaceError.unsupported("app group container unavailable: \(identifier)") } return url + #else + throw WorkspaceError.unsupported("app group container unavailable: \(identifier)") + #endif case let .url(url): return url } diff --git a/Sources/Workspace/FS/SecurityScopedFilesystem.swift b/Sources/Workspace/FS/SecurityScopedFilesystem.swift index 5f4d226..86a4351 100644 --- a/Sources/Workspace/FS/SecurityScopedFilesystem.swift +++ b/Sources/Workspace/FS/SecurityScopedFilesystem.swift @@ -1,3 +1,4 @@ +#if canImport(Darwin) import Foundation /// A disk-backed filesystem rooted at a security-scoped URL. @@ -257,3 +258,4 @@ public final class SecurityScopedFilesystem: FileSystem, @unchecked Sendable { private static let bookmarkResolutionOptions: URL.BookmarkResolutionOptions = [] #endif } +#endif diff --git a/Tests/WorkspaceTests/FilesystemTests.swift b/Tests/WorkspaceTests/FilesystemTests.swift index eaa6d11..e145b50 100644 --- a/Tests/WorkspaceTests/FilesystemTests.swift +++ b/Tests/WorkspaceTests/FilesystemTests.swift @@ -35,11 +35,13 @@ private enum FilesystemTestSupport { } } +#if canImport(Darwin) private final class NilAppGroupFileManager: FileManager { override func containerURL(forSecurityApplicationGroupIdentifier groupIdentifier: String) -> URL? { nil } } +#endif extension Tag { @Tag static var permissions: Self @@ -731,6 +733,7 @@ struct FilesystemTests { #expect(error.description.contains("invalid app group identifier")) } + #if canImport(Darwin) do { _ = try SandboxFilesystem( root: .appGroup("group.workspace.tests.missing"), @@ -740,6 +743,14 @@ struct FilesystemTests { } catch let error as WorkspaceError { #expect(error.description.contains("app group container unavailable")) } + #else + do { + _ = try SandboxFilesystem(root: .appGroup("group.workspace.tests.missing")) + Issue.record("expected unavailable app group rejection") + } catch let error as WorkspaceError { + #expect(error.description.contains("app group container unavailable")) + } + #endif let firstRoot = try FilesystemTestSupport.makeTempDirectory(prefix: "SandboxFirst") defer { FilesystemTestSupport.removeDirectory(firstRoot) } From e5ec091df523630c1e38b109314699c9726818d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 03:35:45 +0000 Subject: [PATCH 2/7] Fix Linux CI toolchain, shell glob semantics, and symlink jail consistency - CI: run the Linux job inside the official swift:6.2-noble container; setup-swift's GPG verification fails on ubuntu runners. - WorkspacePath.globToRegex: * and ? no longer cross path separators, ** matches recursively, and [!abc] negation is translated correctly. Previously /docs/*.txt also matched /docs/sub/deep.txt. - ReadWriteFilesystem: entry-level operations (stat, exists, remove, move source, readSymlink) now use lstat semantics behind the same parent containment check. Symlinks whose targets lie outside the root were previously unstattable and undeletable, and exists() bypassed the jail entirely, following links and acting as an existence oracle for arbitrary external paths. Content operations still refuse to dereference an escaping link. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01G29SXqzHqDycqXGa4dRr4x --- .github/workflows/pr-tests.yml | 14 ++- README.md | 3 + .../Workspace/FS/ReadWriteFilesystem.swift | 54 ++++++++--- Sources/Workspace/Support/WorkspacePath.swift | 23 ++++- Tests/WorkspaceTests/FilesystemTests.swift | 96 +++++++++++++++++++ 5 files changed, 170 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index e7f38b1..fce3bc0 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -19,12 +19,16 @@ jobs: swift-tests: name: Swift Tests (${{ matrix.os }}) runs-on: ${{ matrix.os }} + # Linux runs inside the official Swift image; macOS installs a toolchain via setup-swift. + container: ${{ matrix.container }} strategy: fail-fast: false matrix: - os: - - macos-latest - - ubuntu-24.04 + include: + - os: macos-latest + container: "" + - os: ubuntu-24.04 + container: swift:6.2-noble timeout-minutes: 15 permissions: @@ -37,6 +41,7 @@ jobs: uses: actions/checkout@v5 - name: Set up Swift + if: runner.os == 'macOS' uses: swift-actions/setup-swift@v3 with: swift-version: "6.2" @@ -68,7 +73,8 @@ jobs: list-tests: failed - name: Publish coverage summary - if: always() + # The Swift Linux container does not ship python3; the macOS job covers the summary. + if: always() && runner.os == 'macOS' shell: bash run: | python3 <<'PY' diff --git a/README.md b/README.md index 05d119e..995febc 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,8 @@ let tree = try await workspace.walkTree("/") let summary = try await workspace.summarizeTree("/") ``` +Glob wildcards use shell semantics: `*` and `?` match within a single path component, `**` matches recursively across components, and character classes support negation (`[!abc]`). + JSON helpers encode and decode through `Codable`: ```swift @@ -308,6 +310,7 @@ let workspace = Workspace(filesystem: filesystem) ## Security Notes - Jail and root enforcement belong to the underlying filesystem implementation. +- `ReadWriteFilesystem` uses `lstat` semantics for entry-level operations (`stat`, `exists`, `remove`, `move` source, `readSymlink`): a symlink is handled as the link itself, never its target, so links pointing outside the root can be inspected and deleted but not read or written through. - Permission checks are additive. They do not replace path normalization or jail enforcement. - If you expose `Workspace` to model-driven or remote callers, the host still needs to define what roots, mounts, and permissions are acceptable. diff --git a/Sources/Workspace/FS/ReadWriteFilesystem.swift b/Sources/Workspace/FS/ReadWriteFilesystem.swift index 0a361ac..4c2e706 100644 --- a/Sources/Workspace/FS/ReadWriteFilesystem.swift +++ b/Sources/Workspace/FS/ReadWriteFilesystem.swift @@ -43,9 +43,12 @@ public final class ReadWriteFilesystem: FileSystem, @unchecked Sendable { } /// See ``FileSystem/stat(path:)``. + /// + /// Uses `lstat` semantics: a symlink is reported as a symlink without resolving its target, + /// so links pointing outside the root can still be inspected. public func stat(path: WorkspacePath) async throws -> FileInfo { let configuration = try requireConfiguration() - let url = try existingURL(for: path, configuration: configuration) + let url = try noFollowURL(for: path, configuration: configuration) let attributes = try fileManager.attributesOfItem(atPath: url.path) let fileType = attributes[.type] as? FileAttributeType @@ -126,15 +129,18 @@ public final class ReadWriteFilesystem: FileSystem, @unchecked Sendable { } /// See ``FileSystem/remove(path:recursive:)``. + /// + /// Uses `lstat` semantics: removing a symlink unlinks the link itself — never its target — + /// so dangling links and links pointing outside the root can be cleaned up. public func remove(path: WorkspacePath, recursive: Bool) async throws { let configuration = try requireConfiguration() - let url = try existingURL(for: path, configuration: configuration) + let url = try noFollowURL(for: path, configuration: configuration) - var isDirectory: ObjCBool = false - let exists = fileManager.fileExists(atPath: url.path, isDirectory: &isDirectory) - guard exists else { return } + guard let attributes = try? fileManager.attributesOfItem(atPath: url.path) else { + return + } - if isDirectory.boolValue, !recursive { + if attributes[.type] as? FileAttributeType == .typeDirectory, !recursive { let contents = try fileManager.contentsOfDirectory(atPath: url.path) if !contents.isEmpty { throw NSError(domain: NSPOSIXErrorDomain, code: Int(ENOTEMPTY)) @@ -147,7 +153,7 @@ public final class ReadWriteFilesystem: FileSystem, @unchecked Sendable { /// See ``FileSystem/move(from:to:)``. public func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { let configuration = try requireConfiguration() - let source = try existingURL(for: sourcePath, configuration: configuration) + let source = try noFollowURL(for: sourcePath, configuration: configuration) let destination = try creationURL(for: destinationPath, configuration: configuration) let parent = destination.deletingLastPathComponent() try fileManager.createDirectory(at: parent, withIntermediateDirectories: true) @@ -159,7 +165,7 @@ public final class ReadWriteFilesystem: FileSystem, @unchecked Sendable { async throws { let configuration = try requireConfiguration() - let source = try existingURL(for: sourcePath, configuration: configuration) + let source = try noFollowURL(for: sourcePath, configuration: configuration) let destination = try creationURL(for: destinationPath, configuration: configuration) let sourceInfo = try stat(path: sourcePath, configuration: configuration) @@ -202,9 +208,12 @@ public final class ReadWriteFilesystem: FileSystem, @unchecked Sendable { } /// See ``FileSystem/readSymlink(path:)``. + /// + /// Uses `lstat` semantics so the target string of a link pointing outside the root can still + /// be read without dereferencing it. public func readSymlink(path: WorkspacePath) async throws -> String { let configuration = try requireConfiguration() - let url = try existingURL(for: path, configuration: configuration) + let url = try noFollowURL(for: path, configuration: configuration) return try fileManager.destinationOfSymbolicLink(atPath: url.path) } @@ -225,14 +234,18 @@ public final class ReadWriteFilesystem: FileSystem, @unchecked Sendable { } /// See ``FileSystem/exists(path:)``. + /// + /// Uses `lstat` semantics behind the same containment check as other entry operations: a + /// symlink exists even when dangling, and its target is never resolved, so this cannot be + /// used to probe paths outside the root. public func exists(path: WorkspacePath) async -> Bool { if path.string.contains("\u{0}") { return false } do { let configuration = try requireConfiguration() - let url = try existingOrPotentialURL(for: path, configuration: configuration) - return fileManager.fileExists(atPath: url.path) + let url = try noFollowURL(for: path, configuration: configuration) + return (try? fileManager.attributesOfItem(atPath: url.path)) != nil } catch { return false } @@ -305,6 +318,23 @@ public final class ReadWriteFilesystem: FileSystem, @unchecked Sendable { return url } + /// Returns a jailed URL for operations on the entry itself (`lstat` semantics): the parent + /// chain is resolved and containment-checked, but a symlink at the final component is not + /// followed. This lets callers stat, remove, or move a symlink whose target lies outside the + /// root without ever dereferencing it. + private func noFollowURL( + for virtualPath: WorkspacePath, + configuration: ConfigurationSnapshot + ) throws -> URL { + let url = try existingOrPotentialURL(for: virtualPath, configuration: configuration) + if virtualPath.isRoot { + try ensureInsideRoot(url, configuration: configuration) + return url + } + try ensureInsideRoot(url.deletingLastPathComponent(), configuration: configuration) + return url + } + /// The parent containment check never resolves the destination's final component, so a /// pre-existing symlink at the destination could redirect follow-on opens (for example an /// append) outside the configured root. Reject destinations whose final component is a @@ -358,7 +388,7 @@ public final class ReadWriteFilesystem: FileSystem, @unchecked Sendable { } private func stat(path: WorkspacePath, configuration: ConfigurationSnapshot) throws -> FileInfo { - let url = try existingURL(for: path, configuration: configuration) + let url = try noFollowURL(for: path, configuration: configuration) let attributes = try fileManager.attributesOfItem(atPath: url.path) let fileType = attributes[.type] as? FileAttributeType diff --git a/Sources/Workspace/Support/WorkspacePath.swift b/Sources/Workspace/Support/WorkspacePath.swift index 282527a..bac3528 100644 --- a/Sources/Workspace/Support/WorkspacePath.swift +++ b/Sources/Workspace/Support/WorkspacePath.swift @@ -184,6 +184,9 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl } /// Converts a shell-style glob pattern into an anchored regular expression pattern. + /// + /// `*` and `?` match within a single path component (they never cross `/`), `**` matches any + /// characters including `/`, and character classes support shell-style negation (`[!abc]`). public static func globToRegex(_ pattern: some StringProtocol) -> String { let pattern = String(pattern) var regex = "^" @@ -192,13 +195,25 @@ public struct WorkspacePath: Sendable, Hashable, Comparable, Codable, Expressibl while index < pattern.endIndex { let char = pattern[index] if char == "*" { - regex += ".*" + let nextIndex = pattern.index(after: index) + if nextIndex < pattern.endIndex, pattern[nextIndex] == "*" { + regex += ".*" + index = nextIndex + } else { + regex += "[^/]*" + } } else if char == "?" { - regex += "." + regex += "[^/]" } else if char == "[" { - if let closeIndex = pattern[index...].firstIndex(of: "]") { + if let closeIndex = pattern[index...].firstIndex(of: "]"), + closeIndex > pattern.index(after: index) + { let range = pattern.index(after: index).. Date: Thu, 9 Jul 2026 03:43:23 +0000 Subject: [PATCH 3/7] Add ranged reads, chunked streaming, exclusive create, and capabilities FileSystem gains four operations, all with default implementations so existing conformances keep compiling: - readFile(path:offset:length:) reads a byte range; ReadWriteFilesystem overrides it with a seeking file handle so large files are not loaded fully into memory. - readFileChunks(path:chunkSize:) streams contents as AsyncThrowingStream chunks; the disk backend reads incrementally. - createFile(path:data:) creates exclusively and fails with EEXIST; the disk backend uses an atomic no-overwrite write and the in-memory backend checks-and-writes inside its actor. - capabilities() advertises optional features (symlinks, hard links, permissions, real-path resolution) so callers can branch instead of probing for unsupported errors. Wrappers (Permissioned, Mountable, Overlay, Sandbox, SecurityScoped) forward the new operations; PermissionedFileSystem maps them onto the existing readFile/writeFile permission operations. Workspace exposes readData(from:offset:length:). Also updates the three glob tests that asserted the old crossing-separator wildcard behavior, and marks the checkout safe for git inside the Linux CI container so the test reporter can run. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01G29SXqzHqDycqXGa4dRr4x --- .github/workflows/pr-tests.yml | 4 + README.md | 8 ++ Sources/Workspace/FS/FileSystem.swift | 86 ++++++++++++++++++ Sources/Workspace/FS/InMemoryFilesystem.swift | 20 +++++ .../Workspace/FS/MountableFilesystem.swift | 42 +++++++++ Sources/Workspace/FS/OverlayFilesystem.swift | 23 +++++ .../Workspace/FS/ReadWriteFilesystem.swift | 87 ++++++++++++++++++ Sources/Workspace/FS/SandboxFilesystem.swift | 23 +++++ .../FS/SecurityScopedFilesystem.swift | 24 +++++ Sources/Workspace/Support/Permissions.swift | 26 ++++++ Sources/Workspace/Workspace.swift | 6 ++ Tests/WorkspaceTests/CoreTests.swift | 5 +- Tests/WorkspaceTests/FilesystemTests.swift | 90 +++++++++++++++++++ Tests/WorkspaceTests/MountingTests.swift | 11 ++- 14 files changed, 452 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index fce3bc0..5f046e2 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -49,6 +49,10 @@ jobs: - name: Show Swift version run: swift --version + - name: Mark workspace safe for git in container + if: runner.os == 'Linux' + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Run test suite shell: bash run: | diff --git a/README.md b/README.md index 995febc..7d1420e 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,14 @@ let summary = try await workspace.summarizeTree("/") Glob wildcards use shell semantics: `*` and `?` match within a single path component, `**` matches recursively across components, and character classes support negation (`[!abc]`). +Ranged reads avoid loading whole files when the backing filesystem supports seeking: + +```swift +let slice = try await workspace.readData(from: "/blob.bin", offset: 1024, length: 4096) +``` + +At the `FileSystem` level, implementations also provide `readFileChunks(path:chunkSize:)` for streaming reads, `createFile(path:data:)` for exclusive creation (fails with `EEXIST`), and `capabilities()` to query optional features (symlinks, hard links, permissions, real-path resolution) without probe-and-catch. + JSON helpers encode and decode through `Codable`: ```swift diff --git a/Sources/Workspace/FS/FileSystem.swift b/Sources/Workspace/FS/FileSystem.swift index cd17ff2..f819a72 100644 --- a/Sources/Workspace/FS/FileSystem.swift +++ b/Sources/Workspace/FS/FileSystem.swift @@ -1,5 +1,11 @@ import Foundation +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + /// Errors produced by workspace path and filesystem operations. public enum WorkspaceError: Error, CustomStringConvertible, Sendable { /// The caller supplied a path that cannot be represented safely. @@ -107,6 +113,27 @@ public struct DirectoryEntry: Sendable, Codable { } } +/// Optional features a filesystem implementation can advertise. +/// +/// Callers can branch on capabilities instead of probing operations and catching +/// ``WorkspaceError/unsupported(_:)``. +public struct FileSystemCapabilities: OptionSet, Sendable { + public let rawValue: Int + + public init(rawValue: Int) { + self.rawValue = rawValue + } + + /// Symbolic links can be created and read. + public static let symlinks = FileSystemCapabilities(rawValue: 1 << 0) + /// Hard links can be created. + public static let hardLinks = FileSystemCapabilities(rawValue: 1 << 1) + /// POSIX permissions can be read and mutated. + public static let permissions = FileSystemCapabilities(rawValue: 1 << 2) + /// ``FileSystem/resolveRealPath(path:)`` resolves symlink chains. + public static let realPathResolution = FileSystemCapabilities(rawValue: 1 << 3) +} + /// Read-only filesystem capabilities. public protocol ReadableFileSystem: AnyObject, Sendable { /// Returns metadata for the entry at `path`. @@ -152,6 +179,15 @@ public protocol FileSystem: WritableFileSystem { func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws /// Resolves the real path of `path`, following symlinks. func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath + /// Returns the optional features this filesystem supports. + func capabilities() async -> FileSystemCapabilities + /// Reads up to `length` bytes starting at `offset`. Passing `nil` reads to the end of the + /// file; an offset at or past the end returns empty data. + func readFile(path: WorkspacePath, offset: UInt64, length: Int?) async throws -> Data + /// Streams the file contents as chunks of at most `chunkSize` bytes. + func readFileChunks(path: WorkspacePath, chunkSize: Int) async throws -> AsyncThrowingStream + /// Writes `data` to a new file, failing with `EEXIST` when an entry already exists at `path`. + func createFile(path: WorkspacePath, data: Data) async throws } extension FileSystem { @@ -193,4 +229,54 @@ extension FileSystem { _ = path throw WorkspaceError.unsupported("real path resolution is not supported by this filesystem") } + + /// The default implementation advertises no optional features. + public func capabilities() async -> FileSystemCapabilities { + [] + } + + /// The default implementation reads the whole file and slices the requested range. + /// Implementations backed by real files should override this with a seek-based read. + public func readFile(path: WorkspacePath, offset: UInt64, length: Int?) async throws -> Data { + if let length, length < 0 { + throw WorkspaceError.unsupported("read length must not be negative") + } + let data = try await readFile(path: path) + guard offset < UInt64(data.count) else { + return Data() + } + let start = data.index(data.startIndex, offsetBy: Int(offset)) + let end = length.map { data.index(start, offsetBy: $0, limitedBy: data.endIndex) ?? data.endIndex } ?? data.endIndex + return Data(data[start.. AsyncThrowingStream { + guard chunkSize > 0 else { + throw WorkspaceError.unsupported("chunk size must be positive") + } + let data = try await readFile(path: path) + return AsyncThrowingStream { continuation in + var index = data.startIndex + while index < data.endIndex { + let end = data.index(index, offsetBy: chunkSize, limitedBy: data.endIndex) ?? data.endIndex + continuation.yield(Data(data[index.. FileSystemCapabilities { + [.symlinks, .hardLinks, .permissions, .realPathResolution] + } + + /// See ``FileSystem/createFile(path:data:)``. Actor isolation makes the existence check and + /// write a single atomic step. + public func createFile(path: WorkspacePath, data: Data) async throws { + if (try? node(at: path, followFinalSymlink: false)) != nil { + throw posixError(EEXIST) + } + try await writeFile(path: path, data: data, append: false) + } + /// See ``FileSystem/writeFile(path:data:append:)``. public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { guard !path.isRoot else { diff --git a/Sources/Workspace/FS/MountableFilesystem.swift b/Sources/Workspace/FS/MountableFilesystem.swift index 7217b32..653ac29 100644 --- a/Sources/Workspace/FS/MountableFilesystem.swift +++ b/Sources/Workspace/FS/MountableFilesystem.swift @@ -151,6 +151,48 @@ public final class MountableFilesystem: FileSystem, @unchecked Sendable { return try await base.readFile(path: path) } + /// See ``FileSystem/readFile(path:offset:length:)``. + public func readFile(path: WorkspacePath, offset: UInt64, length: Int?) async throws -> Data { + if let resolved = resolveMounted(path: path, mounts: mountsSnapshot()) { + return try await resolved.filesystem.readFile( + path: resolved.relativePath, + offset: offset, + length: length + ) + } + return try await base.readFile(path: path, offset: offset, length: length) + } + + /// See ``FileSystem/readFileChunks(path:chunkSize:)``. + public func readFileChunks( + path: WorkspacePath, + chunkSize: Int + ) async throws -> AsyncThrowingStream { + if let resolved = resolveMounted(path: path, mounts: mountsSnapshot()) { + return try await resolved.filesystem.readFileChunks( + path: resolved.relativePath, + chunkSize: chunkSize + ) + } + return try await base.readFileChunks(path: path, chunkSize: chunkSize) + } + + /// See ``FileSystem/createFile(path:data:)``. + public func createFile(path: WorkspacePath, data: Data) async throws { + let resolved = resolveWritable(path: path, mounts: mountsSnapshot()) + try await resolved.filesystem.createFile(path: resolved.relativePath, data: data) + } + + /// See ``FileSystem/capabilities()``. Reports the intersection of the base filesystem's and + /// every mount's capabilities, since a path may resolve to any of them. + public func capabilities() async -> FileSystemCapabilities { + var combined = await base.capabilities() + for mount in mountsSnapshot() { + combined.formIntersection(await mount.filesystem.capabilities()) + } + return combined + } + /// See ``FileSystem/writeFile(path:data:append:)``. public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { let resolved = resolveWritable(path: path, mounts: mountsSnapshot()) diff --git a/Sources/Workspace/FS/OverlayFilesystem.swift b/Sources/Workspace/FS/OverlayFilesystem.swift index 5c38bab..5f9fc4d 100644 --- a/Sources/Workspace/FS/OverlayFilesystem.swift +++ b/Sources/Workspace/FS/OverlayFilesystem.swift @@ -66,6 +66,29 @@ public actor OverlayFilesystem: FileSystem { try await overlay.readFile(path: path) } + /// See ``FileSystem/readFile(path:offset:length:)``. + public func readFile(path: WorkspacePath, offset: UInt64, length: Int?) async throws -> Data { + try await overlay.readFile(path: path, offset: offset, length: length) + } + + /// See ``FileSystem/readFileChunks(path:chunkSize:)``. + public func readFileChunks( + path: WorkspacePath, + chunkSize: Int + ) async throws -> AsyncThrowingStream { + try await overlay.readFileChunks(path: path, chunkSize: chunkSize) + } + + /// See ``FileSystem/createFile(path:data:)``. + public func createFile(path: WorkspacePath, data: Data) async throws { + try await overlay.createFile(path: path, data: data) + } + + /// See ``FileSystem/capabilities()``. + public func capabilities() async -> FileSystemCapabilities { + await overlay.capabilities() + } + /// See ``FileSystem/writeFile(path:data:append:)``. public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { try await overlay.writeFile(path: path, data: data, append: append) diff --git a/Sources/Workspace/FS/ReadWriteFilesystem.swift b/Sources/Workspace/FS/ReadWriteFilesystem.swift index 4c2e706..44de74e 100644 --- a/Sources/Workspace/FS/ReadWriteFilesystem.swift +++ b/Sources/Workspace/FS/ReadWriteFilesystem.swift @@ -1,5 +1,11 @@ import Foundation +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + /// A disk-backed filesystem rooted at a concrete directory on the host filesystem. /// /// Paths are resolved relative to the configured root and constrained so callers cannot escape that root @@ -103,6 +109,87 @@ public final class ReadWriteFilesystem: FileSystem, @unchecked Sendable { return try Data(contentsOf: url) } + /// See ``FileSystem/capabilities()``. + public func capabilities() async -> FileSystemCapabilities { + [.symlinks, .hardLinks, .permissions, .realPathResolution] + } + + /// See ``FileSystem/readFile(path:offset:length:)``. Reads only the requested range using a + /// seeking file handle instead of loading the whole file. + public func readFile(path: WorkspacePath, offset: UInt64, length: Int?) async throws -> Data { + if let length, length < 0 { + throw WorkspaceError.unsupported("read length must not be negative") + } + let configuration = try requireConfiguration() + let url = try existingURL(for: path, configuration: configuration) + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + let size = try handle.seekToEnd() + guard offset < size else { + return Data() + } + try handle.seek(toOffset: offset) + if let length { + return try handle.read(upToCount: length) ?? Data() + } + return try handle.readToEnd() ?? Data() + } + + /// See ``FileSystem/readFileChunks(path:chunkSize:)``. Streams the file incrementally so + /// large files never need to be fully resident. + public func readFileChunks( + path: WorkspacePath, + chunkSize: Int + ) async throws -> AsyncThrowingStream { + guard chunkSize > 0 else { + throw WorkspaceError.unsupported("chunk size must be positive") + } + let configuration = try requireConfiguration() + let url = try existingURL(for: path, configuration: configuration) + // Validate readability up front so the caller sees open errors directly. + let probe = try FileHandle(forReadingFrom: url) + try probe.close() + + return AsyncThrowingStream { continuation in + let reader = Task.detached { + do { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + while !Task.isCancelled { + guard let chunk = try handle.read(upToCount: chunkSize), !chunk.isEmpty else { + break + } + continuation.yield(chunk) + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in + reader.cancel() + } + } + } + + /// See ``FileSystem/createFile(path:data:)``. Uses an exclusive write so concurrent creators + /// cannot overwrite each other. + public func createFile(path: WorkspacePath, data: Data) async throws { + let configuration = try requireConfiguration() + let url = try creationURL(for: path, configuration: configuration) + let parent = url.deletingLastPathComponent() + try fileManager.createDirectory(at: parent, withIntermediateDirectories: true) + do { + try data.write(to: url, options: [.withoutOverwriting]) + } catch { + let nsError = error as NSError + if nsError.domain == NSCocoaErrorDomain, nsError.code == CocoaError.fileWriteFileExists.rawValue { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(EEXIST)) + } + throw error + } + } + /// See ``FileSystem/writeFile(path:data:append:)``. public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { let configuration = try requireConfiguration() diff --git a/Sources/Workspace/FS/SandboxFilesystem.swift b/Sources/Workspace/FS/SandboxFilesystem.swift index 2666fba..eb73196 100644 --- a/Sources/Workspace/FS/SandboxFilesystem.swift +++ b/Sources/Workspace/FS/SandboxFilesystem.swift @@ -45,6 +45,29 @@ public final class SandboxFilesystem: FileSystem, Sendable { try await backing.readFile(path: path) } + /// See ``FileSystem/readFile(path:offset:length:)``. + public func readFile(path: WorkspacePath, offset: UInt64, length: Int?) async throws -> Data { + try await backing.readFile(path: path, offset: offset, length: length) + } + + /// See ``FileSystem/readFileChunks(path:chunkSize:)``. + public func readFileChunks( + path: WorkspacePath, + chunkSize: Int + ) async throws -> AsyncThrowingStream { + try await backing.readFileChunks(path: path, chunkSize: chunkSize) + } + + /// See ``FileSystem/createFile(path:data:)``. + public func createFile(path: WorkspacePath, data: Data) async throws { + try await backing.createFile(path: path, data: data) + } + + /// See ``FileSystem/capabilities()``. + public func capabilities() async -> FileSystemCapabilities { + await backing.capabilities() + } + /// See ``FileSystem/writeFile(path:data:append:)``. public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { try await backing.writeFile(path: path, data: data, append: append) diff --git a/Sources/Workspace/FS/SecurityScopedFilesystem.swift b/Sources/Workspace/FS/SecurityScopedFilesystem.swift index 86a4351..2c2c519 100644 --- a/Sources/Workspace/FS/SecurityScopedFilesystem.swift +++ b/Sources/Workspace/FS/SecurityScopedFilesystem.swift @@ -143,6 +143,30 @@ public final class SecurityScopedFilesystem: FileSystem, @unchecked Sendable { try await backing.readFile(path: path) } + /// See ``FileSystem/readFile(path:offset:length:)``. + public func readFile(path: WorkspacePath, offset: UInt64, length: Int?) async throws -> Data { + try await backing.readFile(path: path, offset: offset, length: length) + } + + /// See ``FileSystem/readFileChunks(path:chunkSize:)``. + public func readFileChunks( + path: WorkspacePath, + chunkSize: Int + ) async throws -> AsyncThrowingStream { + try await backing.readFileChunks(path: path, chunkSize: chunkSize) + } + + /// See ``FileSystem/createFile(path:data:)``. + public func createFile(path: WorkspacePath, data: Data) async throws { + try ensureWritable() + try await backing.createFile(path: path, data: data) + } + + /// See ``FileSystem/capabilities()``. + public func capabilities() async -> FileSystemCapabilities { + await backing.capabilities() + } + /// See ``FileSystem/writeFile(path:data:append:)``. public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { try ensureWritable() diff --git a/Sources/Workspace/Support/Permissions.swift b/Sources/Workspace/Support/Permissions.swift index 649e17f..3e22247 100644 --- a/Sources/Workspace/Support/Permissions.swift +++ b/Sources/Workspace/Support/Permissions.swift @@ -149,6 +149,32 @@ public final class PermissionedFileSystem: FileSystem, Sendable { return try await base.readFile(path: path) } + /// See ``FileSystem/readFile(path:offset:length:)``. + public func readFile(path: WorkspacePath, offset: UInt64, length: Int?) async throws -> Data { + try await authorize(.init(operation: .readFile, path: path)) + return try await base.readFile(path: path, offset: offset, length: length) + } + + /// See ``FileSystem/readFileChunks(path:chunkSize:)``. + public func readFileChunks( + path: WorkspacePath, + chunkSize: Int + ) async throws -> AsyncThrowingStream { + try await authorize(.init(operation: .readFile, path: path)) + return try await base.readFileChunks(path: path, chunkSize: chunkSize) + } + + /// See ``FileSystem/createFile(path:data:)``. + public func createFile(path: WorkspacePath, data: Data) async throws { + try await authorize(.init(operation: .writeFile, path: path)) + try await base.createFile(path: path, data: data) + } + + /// See ``FileSystem/capabilities()``. + public func capabilities() async -> FileSystemCapabilities { + await base.capabilities() + } + /// See ``FileSystem/writeFile(path:data:append:)``. public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { try await authorize(.init(operation: .writeFile, path: path, append: append)) diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index d82df07..c3e072d 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -104,6 +104,12 @@ public actor Workspace { try await filesystem.readFile(path: path) } + /// Reads up to `length` bytes starting at `offset` without loading the whole file when the + /// backing filesystem supports ranged reads. + public func readData(from path: WorkspacePath, offset: UInt64, length: Int? = nil) async throws -> Data { + try await filesystem.readFile(path: path, offset: offset, length: length) + } + /// Writes raw file data to the workspace, replacing any existing contents. public func writeData(_ data: Data, to path: WorkspacePath) async throws { try await ensureLoaded() diff --git a/Tests/WorkspaceTests/CoreTests.swift b/Tests/WorkspaceTests/CoreTests.swift index 83ba09f..bc24f38 100644 --- a/Tests/WorkspaceTests/CoreTests.swift +++ b/Tests/WorkspaceTests/CoreTests.swift @@ -1188,8 +1188,11 @@ struct CoreTests { #expect(WorkspacePath.basename("/") == "/") #expect(WorkspacePath.dirname("/") == .root) #expect(WorkspacePath.join("/base", "/override") == "/override") - #expect(WorkspacePath.globToRegex("file?.[ch]") == "^file.\\.[ch]$") + #expect(WorkspacePath.globToRegex("file?.[ch]") == "^file[^/]\\.[ch]$") #expect(WorkspacePath.globToRegex("file[") == "^file\\[$") + #expect(WorkspacePath.globToRegex("*.txt") == "^[^/]*\\.txt$") + #expect(WorkspacePath.globToRegex("**/*.txt") == "^.*/[^/]*\\.txt$") + #expect(WorkspacePath.globToRegex("[!ab]") == "^[^ab]$") let encoded = try JSONEncoder().encode(WorkspacePath(normalizing: "/tmp/../file.txt")) #expect(try JSONDecoder().decode(WorkspacePath.self, from: encoded) == "/file.txt") diff --git a/Tests/WorkspaceTests/FilesystemTests.swift b/Tests/WorkspaceTests/FilesystemTests.swift index 3509da1..e4eaa2d 100644 --- a/Tests/WorkspaceTests/FilesystemTests.swift +++ b/Tests/WorkspaceTests/FilesystemTests.swift @@ -273,6 +273,96 @@ struct FilesystemTests { #expect(try await filesystem.glob(pattern: "/docs/**/*.txt", currentDirectory: "/").contains("/docs/sub/deep.txt")) } + @Test(.tags(.readWrite)) + func `ranged and chunked reads return consistent slices`() async throws { + let root = try FilesystemTestSupport.makeTempDirectory(prefix: "FilesystemRanged") + defer { FilesystemTestSupport.removeDirectory(root) } + + let filesystem = try ReadWriteFilesystem(rootDirectory: root) + let payload = Data("0123456789".utf8) + try await filesystem.writeFile(path: "/data.bin", data: payload, append: false) + + #expect(try await filesystem.readFile(path: "/data.bin", offset: 0, length: nil) == payload) + #expect(try await filesystem.readFile(path: "/data.bin", offset: 3, length: 4) == Data("3456".utf8)) + #expect(try await filesystem.readFile(path: "/data.bin", offset: 8, length: 10) == Data("89".utf8)) + #expect(try await filesystem.readFile(path: "/data.bin", offset: 42, length: nil) == Data()) + + var chunks: [Data] = [] + for try await chunk in try await filesystem.readFileChunks(path: "/data.bin", chunkSize: 3) { + chunks.append(chunk) + } + #expect(chunks.map(\.count) == [3, 3, 3, 1]) + #expect(chunks.reduce(Data(), +) == payload) + + // The protocol's default implementation used by the in-memory backend agrees. + let memory = InMemoryFilesystem() + try await memory.writeFile(path: "/data.bin", data: payload, append: false) + #expect(try await memory.readFile(path: "/data.bin", offset: 3, length: 4) == Data("3456".utf8)) + #expect(try await memory.readFile(path: "/data.bin", offset: 42, length: 1) == Data()) + + var memoryChunks: [Data] = [] + for try await chunk in try await memory.readFileChunks(path: "/data.bin", chunkSize: 4) { + memoryChunks.append(chunk) + } + #expect(memoryChunks.reduce(Data(), +) == payload) + } + + @Test(.tags(.readWrite)) + func `createFile is exclusive on disk and in memory`() async throws { + let root = try FilesystemTestSupport.makeTempDirectory(prefix: "FilesystemCreate") + defer { FilesystemTestSupport.removeDirectory(root) } + + let filesystem = try ReadWriteFilesystem(rootDirectory: root) + try await filesystem.createFile(path: "/fresh/new.txt", data: Data("one".utf8)) + #expect(try await filesystem.readFile(path: "/fresh/new.txt") == Data("one".utf8)) + + do { + try await filesystem.createFile(path: "/fresh/new.txt", data: Data("two".utf8)) + Issue.record("expected EEXIST for existing file") + } catch let error as NSError { + #expect(error.domain == NSPOSIXErrorDomain) + #expect(error.code == Int(EEXIST)) + } + #expect(try await filesystem.readFile(path: "/fresh/new.txt") == Data("one".utf8)) + + let memory = InMemoryFilesystem() + try await memory.createFile(path: "/new.txt", data: Data("one".utf8)) + do { + try await memory.createFile(path: "/new.txt", data: Data("two".utf8)) + Issue.record("expected EEXIST for existing file") + } catch let error as NSError { + #expect(error.code == Int(EEXIST)) + } + #expect(try await memory.readFile(path: "/new.txt") == Data("one".utf8)) + } + + @Test(.tags(.permissions)) + func `capabilities are advertised and forwarded through wrappers`() async throws { + let memory = InMemoryFilesystem() + let memoryCapabilities = await memory.capabilities() + #expect(memoryCapabilities.contains(.symlinks)) + #expect(memoryCapabilities.contains(.permissions)) + + let permissioned = PermissionedFileSystem( + base: memory, + authorizer: PermissionAuthorizer { _ in .allow } + ) + #expect(await permissioned.capabilities() == memoryCapabilities) + + let denied = PermissionedFileSystem( + base: memory, + authorizer: PermissionAuthorizer { request in + request.operation == .writeFile ? .deny(message: "no writes") : .allow + } + ) + do { + try await denied.createFile(path: "/x.txt", data: Data()) + Issue.record("expected createFile denial through writeFile permission") + } catch let error as WorkspaceError { + #expect(error.description.contains("no writes")) + } + } + @Test(.tags(.inMemory)) func `glob character classes support shell negation`() async throws { let filesystem = InMemoryFilesystem() diff --git a/Tests/WorkspaceTests/MountingTests.swift b/Tests/WorkspaceTests/MountingTests.swift index d02d09b..f6edf33 100644 --- a/Tests/WorkspaceTests/MountingTests.swift +++ b/Tests/WorkspaceTests/MountingTests.swift @@ -158,7 +158,10 @@ struct MountingTests { let globbed = try await filesystem.glob(pattern: "/docs/*.txt", currentDirectory: "/") #expect(globbed.contains("/docs/local.txt")) - #expect(globbed.contains("/docs/external/guide.txt")) + #expect(!globbed.contains("/docs/external/guide.txt")) + + let nested = try await filesystem.glob(pattern: "/docs/**", currentDirectory: "/") + #expect(nested.contains("/docs/external/guide.txt")) try await filesystem.remove(path: "/docs/external/new.txt", recursive: false) #expect(!(await mountedDocs.exists(path: "/new.txt"))) @@ -291,6 +294,10 @@ struct MountingTests { let matches = try await filesystem.glob(pattern: "/*.txt", currentDirectory: "/") #expect(matches.contains("/a.txt")) - #expect(matches.contains("/m/b.txt")) + #expect(!matches.contains("/m/b.txt")) + + let recursive = try await filesystem.glob(pattern: "/**.txt", currentDirectory: "/") + #expect(recursive.contains("/a.txt")) + #expect(recursive.contains("/m/b.txt")) } } From f21bd51aed2fdc4e8900e96fd90bdae583cda2bc Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 03:50:35 +0000 Subject: [PATCH 4/7] Rewrite OverlayFilesystem as a lazy copy-on-write overlay The previous implementation eagerly copied the entire source tree, contents included, into memory at configure time, so overlaying a real project cost memory and startup time proportional to the whole tree. The overlay is now a merged view of two layers: reads pass through lazily to the source directory, mutations land in an in-memory upper layer with on-demand copy-up (append copies the file up, whole-tree moves and copies materialize the affected subtree), and deletions are whiteout cuts that hide lower entries without touching the disk. Directory listings merge both layers with upper entries shadowing lower ones, re-created directories do not resurrect hidden lower children, symlinks resolve across layers at the final component, and stat passes through source permissions and modification dates (previously mtimes were lost on import). reload() and configure() drop the upper layer and whiteouts in O(1) instead of re-importing. Behavioral note: files that change on disk are now visible through the overlay immediately, since reads are pass-through rather than a configure-time snapshot. Also fixes the escaped-separator expectation in the globToRegex test. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01G29SXqzHqDycqXGa4dRr4x --- README.md | 4 +- Sources/Workspace/FS/OverlayFilesystem.swift | 490 +++++++++++++++---- Tests/WorkspaceTests/CoreTests.swift | 2 +- Tests/WorkspaceTests/FilesystemTests.swift | 95 ++++ 4 files changed, 496 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 7d1420e..e4cc013 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Many agent and tooling flows need more than plain disk I/O: - `FileSystem`: low-level protocol for custom filesystem backends - `ReadWriteFilesystem`: real disk access rooted to a configured directory - `InMemoryFilesystem`: fully in-memory filesystem for isolated workspaces and tests -- `OverlayFilesystem`: snapshot a disk root and keep writes in memory +- `OverlayFilesystem`: lazy copy-on-write view of a disk root — reads pass through, writes stay in memory - `MountableFilesystem`: compose multiple filesystems under one virtual tree - `PermissionedFileSystem`: wrap any filesystem with operation-level approvals - `SandboxFilesystem`: convenience wrapper for app sandbox roots @@ -263,6 +263,8 @@ let preview = try await workspace.summarizeTree("/Sources", maxDepth: 2) try await workspace.writeFile("/SCRATCH.md", content: "overlay-only change\n") ``` +The overlay is lazy: nothing is copied at configuration time, reads pass through to the source directory (so files that change on disk are visible immediately), and only mutated entries are copied up into memory. Deletions are recorded as whiteouts that hide the source entry without touching the disk. `reload()` discards all overlay writes and whiteouts. + ### Mounted Workspaces ```swift diff --git a/Sources/Workspace/FS/OverlayFilesystem.swift b/Sources/Workspace/FS/OverlayFilesystem.swift index 5f9fc4d..01359ef 100644 --- a/Sources/Workspace/FS/OverlayFilesystem.swift +++ b/Sources/Workspace/FS/OverlayFilesystem.swift @@ -1,203 +1,507 @@ import Foundation -/// A filesystem that snapshots a disk directory into an in-memory overlay. +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#endif + +/// A copy-on-write overlay over a disk directory. +/// +/// Reads pass through lazily to the source directory (the *lower* layer); mutations land in an +/// in-memory *upper* layer, leaving the source directory untouched. Nothing is copied at +/// configuration time: a file's bytes enter memory only when it is read through the overlay or +/// copied up by a mutation. Deletions are recorded as whiteouts that hide the lower entry, and +/// directory listings merge both layers with upper entries shadowing lower ones. /// -/// Mutations apply only to the overlay, leaving the source directory untouched until the overlay is -/// rebuilt. +/// `reload()` (or `configure(rootDirectory:)`) discards the upper layer and whiteouts, so the +/// overlay reflects the source directory's current state again. public actor OverlayFilesystem: FileSystem { private let fileManager: FileManager - private var overlay: InMemoryFilesystem + private var upper: InMemoryFilesystem + private var lower: ReadWriteFilesystem? private var rootURL: URL? - private var configurationVersion = 0 + /// Paths whose lower-layer entries (including everything beneath them) are hidden from the + /// merged view. Creating a new entry at a cut path shadows it in the upper layer; the cut + /// keeps hiding the original lower content. + private var cuts: Set = [] /// Creates an unconfigured overlay filesystem. public init(fileManager: FileManager = .default) { self.fileManager = fileManager - overlay = InMemoryFilesystem() + upper = InMemoryFilesystem() } /// Creates and configures an overlay for `rootDirectory`. public init(rootDirectory: URL, fileManager: FileManager = .default) async throws { self.fileManager = fileManager - overlay = InMemoryFilesystem() + upper = InMemoryFilesystem() try await configure(rootDirectory: rootDirectory) } /// See ``FileSystem/configure(rootDirectory:)``. public func configure(rootDirectory: URL) async throws { - let standardizedRoot = rootDirectory.standardizedFileURL - configurationVersion += 1 - let version = configurationVersion - rootURL = standardizedRoot - - let newOverlay = try await makeOverlay(from: standardizedRoot) - guard configurationVersion == version, rootURL == standardizedRoot else { - return - } - overlay = newOverlay + rootURL = rootDirectory.standardizedFileURL + try resetLayers() } - /// Rebuilds the overlay from the configured root directory. + /// Discards overlay writes and whiteouts so reads reflect the source directory again. public func reload() async throws { - guard let currentRoot = rootURL else { + guard rootURL != nil else { throw WorkspaceError.unsupported("overlay filesystem requires rootDirectory") } + try resetLayers() + } - let version = configurationVersion - let newOverlay = try await makeOverlay(from: currentRoot) - guard configurationVersion == version, rootURL == currentRoot else { + private func resetLayers() throws { + upper = InMemoryFilesystem() + cuts = [] + guard let rootURL else { + lower = nil return } - overlay = newOverlay + var isDirectory: ObjCBool = false + if fileManager.fileExists(atPath: rootURL.path, isDirectory: &isDirectory), isDirectory.boolValue { + lower = try ReadWriteFilesystem(rootDirectory: rootURL, fileManager: fileManager) + } else { + lower = nil + } } + // MARK: - Reads + /// See ``FileSystem/stat(path:)``. public func stat(path: WorkspacePath) async throws -> FileInfo { - try await overlay.stat(path: path) + guard let info = await mergedEntry(path) else { + throw posixError(ENOENT) + } + var adjusted = info + adjusted.path = path + return adjusted } /// See ``FileSystem/listDirectory(path:)``. public func listDirectory(path: WorkspacePath) async throws -> [DirectoryEntry] { - try await overlay.listDirectory(path: path) + let upperInfo = await upperEntry(path) + let lowerInfo = await lowerVisibleEntry(path) + guard upperInfo != nil || lowerInfo != nil else { + throw posixError(ENOENT) + } + + var byName: [String: DirectoryEntry] = [:] + if let lowerInfo, upperInfo == nil || upperInfo?.kind == .directory { + guard lowerInfo.kind == .directory || upperInfo != nil else { + throw posixError(ENOTDIR) + } + if lowerInfo.kind == .directory, let lower { + for entry in try await lower.listDirectory(path: path) + where !isCut(path.appending(entry.name)) { + byName[entry.name] = entry + } + } + } + if let upperInfo { + guard upperInfo.kind == .directory else { + throw posixError(ENOTDIR) + } + for entry in try await upper.listDirectory(path: path) { + byName[entry.name] = entry + } + } + return byName.values.sorted { $0.name < $1.name } } /// See ``FileSystem/readFile(path:)``. public func readFile(path: WorkspacePath) async throws -> Data { - try await overlay.readFile(path: path) + let target = try await resolveMergedFinalSymlink(path) + if await upperEntry(target) != nil { + return try await upper.readFile(path: target) + } + guard let info = await lowerVisibleEntry(target), let lower else { + throw posixError(ENOENT) + } + guard info.kind != .directory else { + throw posixError(EISDIR) + } + return try await lower.readFile(path: target) } - /// See ``FileSystem/readFile(path:offset:length:)``. - public func readFile(path: WorkspacePath, offset: UInt64, length: Int?) async throws -> Data { - try await overlay.readFile(path: path, offset: offset, length: length) + /// See ``FileSystem/readSymlink(path:)``. + public func readSymlink(path: WorkspacePath) async throws -> String { + if await upperEntry(path) != nil { + return try await upper.readSymlink(path: path) + } + guard let info = await lowerVisibleEntry(path), let lower else { + throw posixError(ENOENT) + } + guard info.kind == .symlink else { + throw posixError(EINVAL) + } + return try await lower.readSymlink(path: path) } - /// See ``FileSystem/readFileChunks(path:chunkSize:)``. - public func readFileChunks( - path: WorkspacePath, - chunkSize: Int - ) async throws -> AsyncThrowingStream { - try await overlay.readFileChunks(path: path, chunkSize: chunkSize) + /// See ``FileSystem/resolveRealPath(path:)``. + public func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath { + let target = try await resolveMergedFinalSymlink(path) + guard await mergedEntry(target) != nil else { + throw posixError(ENOENT) + } + return target } - /// See ``FileSystem/createFile(path:data:)``. - public func createFile(path: WorkspacePath, data: Data) async throws { - try await overlay.createFile(path: path, data: data) + /// See ``FileSystem/exists(path:)``. + public func exists(path: WorkspacePath) async -> Bool { + guard let target = try? await resolveMergedFinalSymlink(path) else { + return false + } + return await mergedEntry(target) != nil + } + + /// See ``FileSystem/glob(pattern:currentDirectory:)``. + public func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { + let normalizedPattern = try WorkspacePath(validating: pattern, relativeTo: currentDirectory) + if !WorkspacePath.containsGlob(normalizedPattern.string) { + return await exists(path: normalizedPattern) ? [normalizedPattern] : [] + } + + let regex = try NSRegularExpression(pattern: WorkspacePath.globToRegex(normalizedPattern.string)) + let paths = await allMergedPaths() + return paths.filter { path in + let range = NSRange(path.string.startIndex.. FileSystemCapabilities { - await overlay.capabilities() + [.symlinks, .hardLinks, .permissions, .realPathResolution] } + // MARK: - Mutations + /// See ``FileSystem/writeFile(path:data:append:)``. public func writeFile(path: WorkspacePath, data: Data, append: Bool) async throws { - try await overlay.writeFile(path: path, data: data, append: append) + guard !path.isRoot else { + throw posixError(EISDIR) + } + let target = try await resolveMergedFinalSymlink(path) + if await upperEntry(target) != nil { + try await upper.writeFile(path: target, data: data, append: append) + return + } + if let info = await lowerVisibleEntry(target), let lower { + guard info.kind != .directory else { + throw posixError(EISDIR) + } + try await ensureUpperDirectories(for: target) + let contents: Data + if append { + contents = try await lower.readFile(path: target) + data + } else { + contents = data + } + try await upper.writeFile(path: target, data: contents, append: false) + try await upper.setPermissions(path: target, permissions: info.permissions) + return + } + try await ensureUpperDirectories(for: target) + try await upper.writeFile(path: target, data: data, append: false) + } + + /// See ``FileSystem/createFile(path:data:)``. + public func createFile(path: WorkspacePath, data: Data) async throws { + guard await mergedEntry(path) == nil else { + throw posixError(EEXIST) + } + try await ensureUpperDirectories(for: path) + try await upper.createFile(path: path, data: data) } /// See ``FileSystem/createDirectory(path:recursive:)``. public func createDirectory(path: WorkspacePath, recursive: Bool) async throws { - try await overlay.createDirectory(path: path, recursive: recursive) + if path.isRoot { + return + } + + var current = WorkspacePath.root + let components = path.components + for (index, component) in components.enumerated() { + current = current.appending(component) + let isLast = index == components.count - 1 + + if let info = await mergedEntry(current) { + guard info.kind == .directory else { + throw posixError(ENOTDIR) + } + if isLast, !recursive { + throw posixError(EEXIST) + } + continue + } + + if !recursive, !isLast { + throw posixError(ENOENT) + } + try await ensureUpperDirectories(for: current) + try await upper.createDirectory(path: current, recursive: false) + } } /// See ``FileSystem/remove(path:recursive:)``. public func remove(path: WorkspacePath, recursive: Bool) async throws { - try await overlay.remove(path: path, recursive: recursive) + guard !path.isRoot else { + throw posixError(EPERM) + } + let upperHas = await upperEntry(path) != nil + let lowerVisible = await lowerVisibleEntry(path) != nil + guard upperHas || lowerVisible else { + return + } + + if !recursive, (await mergedEntry(path))?.kind == .directory { + let children = try await listDirectory(path: path) + if !children.isEmpty { + throw posixError(ENOTEMPTY) + } + } + + if upperHas { + try await upper.remove(path: path, recursive: true) + } + if lowerVisible { + // Deeper cuts are redundant once this path is cut. + cuts = cuts.filter { !$0.string.hasPrefix(path.string + "/") } + cuts.insert(path) + } } /// See ``FileSystem/move(from:to:)``. public func move(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath) async throws { - try await overlay.move(from: sourcePath, to: destinationPath) + if sourcePath == destinationPath { + return + } + guard let sourceInfo = await mergedEntry(sourcePath) else { + throw posixError(ENOENT) + } + if sourceInfo.kind == .directory, destinationPath.string.hasPrefix(sourcePath.string + "/") { + throw posixError(EINVAL) + } + guard await mergedEntry(destinationPath) == nil else { + throw posixError(EEXIST) + } + + try await materializeTree(sourcePath) + try await ensureUpperDirectories(for: destinationPath) + try await upper.move(from: sourcePath, to: destinationPath) + if await lowerVisibleEntry(sourcePath) != nil { + cuts = cuts.filter { !$0.string.hasPrefix(sourcePath.string + "/") } + cuts.insert(sourcePath) + } } /// See ``FileSystem/copy(from:to:recursive:)``. public func copy(from sourcePath: WorkspacePath, to destinationPath: WorkspacePath, recursive: Bool) async throws { - try await overlay.copy(from: sourcePath, to: destinationPath, recursive: recursive) + guard let sourceInfo = await mergedEntry(sourcePath) else { + throw posixError(ENOENT) + } + if sourceInfo.kind == .directory, !recursive { + throw posixError(EISDIR) + } + guard await mergedEntry(destinationPath) == nil else { + throw posixError(EEXIST) + } + + try await materializeTree(sourcePath) + try await ensureUpperDirectories(for: destinationPath) + try await upper.copy(from: sourcePath, to: destinationPath, recursive: recursive) } /// See ``FileSystem/createSymlink(path:target:)``. public func createSymlink(path: WorkspacePath, target: String) async throws { - try await overlay.createSymlink(path: path, target: target) + _ = try WorkspacePath(validating: target, relativeTo: path.dirname) + guard await mergedEntry(path) == nil else { + throw posixError(EEXIST) + } + try await ensureUpperDirectories(for: path) + try await upper.createSymlink(path: path, target: target) } /// See ``FileSystem/createHardLink(path:target:)``. public func createHardLink(path: WorkspacePath, target: WorkspacePath) async throws { - try await overlay.createHardLink(path: path, target: target) - } + guard let targetInfo = await mergedEntry(target) else { + throw posixError(ENOENT) + } + guard targetInfo.kind != .directory else { + throw posixError(EPERM) + } + guard await mergedEntry(path) == nil else { + throw posixError(EEXIST) + } - /// See ``FileSystem/readSymlink(path:)``. - public func readSymlink(path: WorkspacePath) async throws -> String { - try await overlay.readSymlink(path: path) + if await upperEntry(target) == nil { + try await copyUpLeaf(target, info: targetInfo) + } + try await ensureUpperDirectories(for: path) + try await upper.createHardLink(path: path, target: target) } /// See ``FileSystem/setPermissions(path:permissions:)``. public func setPermissions(path: WorkspacePath, permissions: POSIXPermissions) async throws { - try await overlay.setPermissions(path: path, permissions: permissions) + if await upperEntry(path) == nil { + guard let info = await lowerVisibleEntry(path) else { + throw posixError(ENOENT) + } + if info.kind == .directory { + try await ensureUpperDirectories(for: path) + try await upper.createDirectory(path: path, recursive: true) + } else { + try await copyUpLeaf(path, info: info) + } + } + try await upper.setPermissions(path: path, permissions: permissions) } - /// See ``FileSystem/resolveRealPath(path:)``. - public func resolveRealPath(path: WorkspacePath) async throws -> WorkspacePath { - try await overlay.resolveRealPath(path: path) + // MARK: - Merged view + + private func isCut(_ path: WorkspacePath) -> Bool { + if cuts.contains(path) { + return true + } + return cuts.contains { path.string.hasPrefix($0.string + "/") } } - /// See ``FileSystem/exists(path:)``. - public func exists(path: WorkspacePath) async -> Bool { - await overlay.exists(path: path) + private func upperEntry(_ path: WorkspacePath) async -> FileInfo? { + try? await upper.stat(path: path) } - /// See ``FileSystem/glob(pattern:currentDirectory:)``. - public func glob(pattern: String, currentDirectory: WorkspacePath) async throws -> [WorkspacePath] { - try await overlay.glob(pattern: pattern, currentDirectory: currentDirectory) + private func lowerVisibleEntry(_ path: WorkspacePath) async -> FileInfo? { + guard let lower, !isCut(path) else { + return nil + } + return try? await lower.stat(path: path) + } + + private func mergedEntry(_ path: WorkspacePath) async -> FileInfo? { + if let info = await upperEntry(path) { + return info + } + return await lowerVisibleEntry(path) + } + + /// Follows a chain of symlinks at the final path component through the merged view, so a + /// link in one layer can point at an entry in the other. + private func resolveMergedFinalSymlink(_ path: WorkspacePath) async throws -> WorkspacePath { + var current = path + var depth = 0 + while depth <= 64 { + guard let info = await mergedEntry(current), info.kind == .symlink else { + return current + } + let target: String + if await upperEntry(current) != nil { + target = try await upper.readSymlink(path: current) + } else if let lower { + target = try await lower.readSymlink(path: current) + } else { + return current + } + current = WorkspacePath(normalizing: target, relativeTo: current.dirname) + depth += 1 + } + throw posixError(ELOOP) } - private func makeOverlay(from rootURL: URL) async throws -> InMemoryFilesystem { - let overlay = InMemoryFilesystem() - guard fileManager.fileExists(atPath: rootURL.path) else { - return overlay + /// Materializes every merged-visible ancestor directory of `path` into the upper layer, + /// preserving lower-layer permissions. Throws `ENOENT` when an ancestor does not exist in + /// the merged view (parents are not invented, matching the in-memory backend). + private func ensureUpperDirectories(for path: WorkspacePath) async throws { + var current = WorkspacePath.root + for component in path.dirname.components { + current = current.appending(component) + if await upperEntry(current) != nil { + continue + } + guard let info = await lowerVisibleEntry(current) else { + throw posixError(ENOENT) + } + guard info.kind == .directory else { + throw posixError(ENOTDIR) + } + try await upper.createDirectory(path: current, recursive: false) + try await upper.setPermissions(path: current, permissions: info.permissions) } + } - let names = try fileManager.contentsOfDirectory(atPath: rootURL.path).sorted() - for name in names { - let childURL = rootURL.appendingPathComponent(name, isDirectory: true) - try await importItem(at: childURL, virtualPath: WorkspacePath.root.appending(name), into: overlay) + /// Copies a lower-layer file or symlink into the upper layer as-is. + private func copyUpLeaf(_ path: WorkspacePath, info: FileInfo) async throws { + guard let lower else { + throw posixError(ENOENT) + } + try await ensureUpperDirectories(for: path) + switch info.kind { + case .symlink: + try await upper.createSymlink(path: path, target: lower.readSymlink(path: path)) + default: + try await upper.writeFile(path: path, data: lower.readFile(path: path), append: false) + try await upper.setPermissions(path: path, permissions: info.permissions) } - return overlay } - private func importItem(at url: URL, virtualPath: WorkspacePath, into overlay: InMemoryFilesystem) async throws { - let attributes = try fileManager.attributesOfItem(atPath: url.path) - let fileType = attributes[.type] as? FileAttributeType - let permissionBits = (attributes[.posixPermissions] as? NSNumber)?.intValue - let permissions = permissionBits.map(POSIXPermissions.init(_:)) + /// Materializes the merged subtree rooted at `path` into the upper layer so a whole-tree + /// mutation (move or copy) can be delegated to the upper backend. + private func materializeTree(_ path: WorkspacePath) async throws { + guard let info = await mergedEntry(path) else { + throw posixError(ENOENT) + } - if fileType == .typeSymbolicLink { - let target = try fileManager.destinationOfSymbolicLink(atPath: url.path) - try await overlay.createSymlink(path: virtualPath, target: target) - if let permissions { - try await overlay.setPermissions(path: virtualPath, permissions: permissions) + if info.kind != .directory { + if await upperEntry(path) == nil { + try await copyUpLeaf(path, info: info) } return } - if fileType == .typeDirectory { - try await overlay.createDirectory(path: virtualPath, recursive: true) - if let permissions { - try await overlay.setPermissions(path: virtualPath, permissions: permissions) - } + if await upperEntry(path) == nil { + try await ensureUpperDirectories(for: path) + try await upper.createDirectory(path: path, recursive: true) + try await upper.setPermissions(path: path, permissions: info.permissions) + } - let children = try fileManager.contentsOfDirectory(atPath: url.path).sorted() - for child in children { - let childURL = url.appendingPathComponent(child, isDirectory: true) - try await importItem(at: childURL, virtualPath: virtualPath.appending(child), into: overlay) + if let lower, let lowerInfo = await lowerVisibleEntry(path), lowerInfo.kind == .directory { + for entry in try await lower.listDirectory(path: path) { + let child = path.appending(entry.name) + if isCut(child) { + continue + } + try await materializeTree(child) } - return } + } - let data = try Data(contentsOf: url) - try await overlay.writeFile(path: virtualPath, data: data, append: false) - if let permissions { - try await overlay.setPermissions(path: virtualPath, permissions: permissions) + private func allMergedPaths() async -> [WorkspacePath] { + var paths: [WorkspacePath] = [.root] + var queue: [WorkspacePath] = [.root] + var index = 0 + while index < queue.count { + let directory = queue[index] + index += 1 + guard let entries = try? await listDirectory(path: directory) else { + continue + } + for entry in entries { + let child = directory.appending(entry.name) + paths.append(child) + if entry.info.kind == .directory { + queue.append(child) + } + } } + return paths + } + + private func posixError(_ code: Int32) -> NSError { + NSError(domain: NSPOSIXErrorDomain, code: Int(code)) } } diff --git a/Tests/WorkspaceTests/CoreTests.swift b/Tests/WorkspaceTests/CoreTests.swift index bc24f38..9721616 100644 --- a/Tests/WorkspaceTests/CoreTests.swift +++ b/Tests/WorkspaceTests/CoreTests.swift @@ -1191,7 +1191,7 @@ struct CoreTests { #expect(WorkspacePath.globToRegex("file?.[ch]") == "^file[^/]\\.[ch]$") #expect(WorkspacePath.globToRegex("file[") == "^file\\[$") #expect(WorkspacePath.globToRegex("*.txt") == "^[^/]*\\.txt$") - #expect(WorkspacePath.globToRegex("**/*.txt") == "^.*/[^/]*\\.txt$") + #expect(WorkspacePath.globToRegex("**/*.txt") == "^.*\\/[^/]*\\.txt$") #expect(WorkspacePath.globToRegex("[!ab]") == "^[^ab]$") let encoded = try JSONEncoder().encode(WorkspacePath(normalizing: "/tmp/../file.txt")) diff --git a/Tests/WorkspaceTests/FilesystemTests.swift b/Tests/WorkspaceTests/FilesystemTests.swift index e4eaa2d..5d4e63b 100644 --- a/Tests/WorkspaceTests/FilesystemTests.swift +++ b/Tests/WorkspaceTests/FilesystemTests.swift @@ -762,6 +762,101 @@ struct FilesystemTests { #expect(!(await filesystem.exists(path: "/scratch/moved.txt"))) } + @Test(.tags(.overlay)) + func `overlay reads pass through lazily and writes stay in memory`() async throws { + let root = try FilesystemTestSupport.makeTempDirectory(prefix: "OverlayLazy") + defer { FilesystemTestSupport.removeDirectory(root) } + + let noteURL = root.appendingPathComponent("note.txt") + try Data("disk".utf8).write(to: noteURL) + + let filesystem = try await OverlayFilesystem(rootDirectory: root) + #expect(try await filesystem.readFile(path: "/note.txt") == Data("disk".utf8)) + + // A file added on disk after configuration is visible without reload: reads are lazy. + try Data("late".utf8).write(to: root.appendingPathComponent("late.txt")) + #expect(try await filesystem.readFile(path: "/late.txt") == Data("late".utf8)) + + // Overlay writes shadow the source without touching it. + try await filesystem.writeFile(path: "/note.txt", data: Data("overlay".utf8), append: false) + #expect(try await filesystem.readFile(path: "/note.txt") == Data("overlay".utf8)) + #expect(try Data(contentsOf: noteURL) == Data("disk".utf8)) + } + + @Test(.tags(.overlay)) + func `overlay deletions hide lower entries and re-creation yields a fresh directory`() async throws { + let root = try FilesystemTestSupport.makeTempDirectory(prefix: "OverlayWhiteout") + defer { FilesystemTestSupport.removeDirectory(root) } + + let dirURL = root.appendingPathComponent("dir", isDirectory: true) + try FileManager.default.createDirectory(at: dirURL, withIntermediateDirectories: true) + try Data("a".utf8).write(to: dirURL.appendingPathComponent("a.txt")) + try Data("b".utf8).write(to: dirURL.appendingPathComponent("b.txt")) + + let filesystem = try await OverlayFilesystem(rootDirectory: root) + + try await filesystem.remove(path: "/dir/a.txt", recursive: false) + #expect(!(await filesystem.exists(path: "/dir/a.txt"))) + #expect((try await filesystem.listDirectory(path: "/dir")).map(\.name) == ["b.txt"]) + #expect(FileManager.default.fileExists(atPath: dirURL.appendingPathComponent("a.txt").path)) + + // Removing the whole directory and re-creating it must not resurrect lower children. + try await filesystem.remove(path: "/dir", recursive: true) + #expect(!(await filesystem.exists(path: "/dir"))) + try await filesystem.createDirectory(path: "/dir", recursive: false) + #expect((try await filesystem.listDirectory(path: "/dir")).isEmpty) + + // reload drops all overlay state. + try await filesystem.reload() + #expect(try await filesystem.readFile(path: "/dir/a.txt") == Data("a".utf8)) + } + + @Test(.tags(.overlay)) + func `overlay copies up on append and merges listings across layers`() async throws { + let root = try FilesystemTestSupport.makeTempDirectory(prefix: "OverlayCopyUp") + defer { FilesystemTestSupport.removeDirectory(root) } + + try Data("one".utf8).write(to: root.appendingPathComponent("log.txt")) + let subURL = root.appendingPathComponent("sub", isDirectory: true) + try FileManager.default.createDirectory(at: subURL, withIntermediateDirectories: true) + try Data("k".utf8).write(to: subURL.appendingPathComponent("keep.txt")) + + let filesystem = try await OverlayFilesystem(rootDirectory: root) + + try await filesystem.writeFile(path: "/log.txt", data: Data(" two".utf8), append: true) + #expect(try await filesystem.readFile(path: "/log.txt") == Data("one two".utf8)) + #expect(try Data(contentsOf: root.appendingPathComponent("log.txt")) == Data("one".utf8)) + + try await filesystem.writeFile(path: "/sub/new.txt", data: Data("n".utf8), append: false) + #expect((try await filesystem.listDirectory(path: "/sub")).map(\.name) == ["keep.txt", "new.txt"]) + + try await filesystem.move(from: "/sub/keep.txt", to: "/sub/moved.txt") + #expect(!(await filesystem.exists(path: "/sub/keep.txt"))) + #expect(try await filesystem.readFile(path: "/sub/moved.txt") == Data("k".utf8)) + #expect(FileManager.default.fileExists(atPath: subURL.appendingPathComponent("keep.txt").path)) + } + + @Test(.tags(.overlay)) + func `overlay stat passes through source metadata for untouched files`() async throws { + let root = try FilesystemTestSupport.makeTempDirectory(prefix: "OverlayMetadata") + defer { FilesystemTestSupport.removeDirectory(root) } + + let fileURL = root.appendingPathComponent("old.txt") + try Data("old".utf8).write(to: fileURL) + let past = Date(timeIntervalSince1970: 946_684_800) + try FileManager.default.setAttributes( + [.modificationDate: past, .posixPermissions: 0o640], + ofItemAtPath: fileURL.path + ) + + let filesystem = try await OverlayFilesystem(rootDirectory: root) + let info = try await filesystem.stat(path: "/old.txt") + + #expect(info.permissions == POSIXPermissions(0o640)) + let mtime = try #require(info.modificationDate) + #expect(abs(mtime.timeIntervalSince(past)) < 1.5) + } + @Test(.tags(.overlay)) func `overlay filesystem reload requires a configured root and treats missing roots as empty`() async throws { let unconfigured = OverlayFilesystem() From 32bd986487f05c66bc892104d413d6d7b7afa760 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 03:58:38 +0000 Subject: [PATCH 5/7] Store snapshots content-addressed; track rollbacks; add log pruning FileCheckpointStore now persists snapshots as manifests plus content-addressed blobs: - File bytes live in per-workspace blobs/ files, written once per unique content, so consecutive checkpoints share unchanged files instead of re-serializing the whole tree, and contents are stored raw instead of base64-inflated inside pretty-printed JSON. - Snapshot manifests carry the tree structure, permissions, and content hashes. Legacy inline-snapshot JSON files still load. - Hashing uses a small dependency-free SHA-256 (FIPS 180-4) with test vectors, keeping the package free of external dependencies. - A missing blob surfaces as the new WorkspaceError.storageCorrupted. Store robustness: - rollback(to:) now records the tree changes it performs as a new `rollback` mutation kind, closing the gap where the mutation log missed exactly the changes made by rollbacks. - CheckpointStore gains pruneMutationRecords(workspaceId:throughSequence:) and Workspace gains pruneMutationHistory(throughSequence:), which bound the mutation log while always retaining the newest record so sequence numbers stay monotonic. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01G29SXqzHqDycqXGa4dRr4x --- README.md | 4 +- Sources/Workspace/CheckpointStore.swift | 172 +++++++++++++++++- Sources/Workspace/FS/FileSystem.swift | 4 + Sources/Workspace/MutationRecord.swift | 1 + Sources/Workspace/Support/SHA256.swift | 102 +++++++++++ Sources/Workspace/Workspace+Checkpoints.swift | 25 +++ .../WorkspaceTests/CheckpointStoreTests.swift | 123 +++++++++++++ Tests/WorkspaceTests/SHA256Tests.swift | 40 ++++ .../WorkspaceCheckpointTests.swift | 40 ++++ .../WorkspaceInternalsTests.swift | 2 + 10 files changed, 509 insertions(+), 4 deletions(-) create mode 100644 Sources/Workspace/Support/SHA256.swift create mode 100644 Tests/WorkspaceTests/SHA256Tests.swift diff --git a/README.md b/README.md index e4cc013..e44e1a5 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ let workspace = Workspace( ) ``` -`Storage.directory(at:)` writes checkpoint and snapshot JSON plus a `mutations.jsonl` append log (one JSON record per line) under `//`. A legacy `mutations.json` array is migrated to JSONL on first access. The store assigns monotonic `sequence` numbers while holding `mutations.lock` (advisory `flock` where the OS supports it), so multiple `Workspace` instances that share a `workspaceId` and store do not collide on mutation sequence. Appends read only the log's final record to derive the next sequence, and a partial trailing line left by a crashed append is skipped on read and truncated before the next append. The current checkpoint head is derived from the parent-id graph (unparented tips), not only from `createdAt` ordering, which reduces surprises when wall clocks differ between processes. Listing mutations still reads the full log; very long histories may need application-level rotation. Coordinating multiple hosts or network disks that do not honor `flock` may still require extra synchronization. +`Storage.directory(at:)` writes checkpoint metadata, snapshot manifests, and a `mutations.jsonl` append log (one JSON record per line) under `//`. Snapshots are content-addressed: file bytes are stored once per unique content in `blobs/`, and each snapshot manifest references them by hash — so consecutive checkpoints share unchanged file content instead of re-serializing the whole tree, and contents are stored raw rather than base64-inflated. Legacy inline-snapshot JSON files and a legacy `mutations.json` array are still read (the latter is migrated to JSONL on first access). Blobs are retained until pruned by future GC tooling. The store assigns monotonic `sequence` numbers while holding `mutations.lock` (advisory `flock` where the OS supports it), so multiple `Workspace` instances that share a `workspaceId` and store do not collide on mutation sequence. Appends read only the log's final record to derive the next sequence, and a partial trailing line left by a crashed append is skipped on read and truncated before the next append. The current checkpoint head is derived from the parent-id graph (unparented tips), not only from `createdAt` ordering, which reduces surprises when wall clocks differ between processes. Listing mutations still reads the full log; very long histories may need application-level rotation. Coordinating multiple hosts or network disks that do not honor `flock` may still require extra synchronization. ### Reads @@ -216,7 +216,7 @@ print(all.count) print(snapshot.rootPath) ``` -Public `restoreSnapshot(_:)` is also tracked as a workspace mutation. Checkpoint rollback uses an internal untracked restore so the rollback is represented by the rollback checkpoint, not by a second restore mutation. +Public `restoreSnapshot(_:)` is also tracked as a workspace mutation. Checkpoint rollback records the tree changes it performs as a `rollback` mutation in addition to the rollback checkpoint, so the mutation log remains a complete account of filesystem changes. Long-running workspaces can bound the log with `pruneMutationHistory(throughSequence:)`, which always retains the newest record so sequence numbers stay monotonic. ### Branch And Merge diff --git a/Sources/Workspace/CheckpointStore.swift b/Sources/Workspace/CheckpointStore.swift index 3ae133b..1d4f104 100644 --- a/Sources/Workspace/CheckpointStore.swift +++ b/Sources/Workspace/CheckpointStore.swift @@ -20,6 +20,9 @@ protocol CheckpointStore: AnyObject, Sendable { @discardableResult func appendMutation(_ mutation: MutationRecord) async throws -> MutationRecord func listMutationRecords(workspaceId: UUID) async throws -> [MutationRecord] + /// Removes mutation records with `sequence <= throughSequence`. The record carrying the + /// highest sequence is always retained so later appends stay monotonic. + func pruneMutationRecords(workspaceId: UUID, throughSequence: Int) async throws } /// An in-memory checkpoint store for tests and ephemeral workspaces. @@ -75,6 +78,18 @@ actor InMemoryCheckpointStore: CheckpointStore { func listMutationRecords(workspaceId: UUID) async throws -> [MutationRecord] { (mutationsByWorkspace[workspaceId] ?? []).sorted { $0.sequence < $1.sequence } } + + func pruneMutationRecords(workspaceId: UUID, throughSequence: Int) async throws { + let records = (mutationsByWorkspace[workspaceId] ?? []).sorted { $0.sequence < $1.sequence } + guard let last = records.last else { + return + } + var kept = records.filter { $0.sequence > throughSequence } + if kept.isEmpty { + kept = [last] + } + mutationsByWorkspace[workspaceId] = kept + } } /// A JSON file-backed checkpoint store. @@ -154,9 +169,39 @@ actor FileCheckpointStore: CheckpointStore { return result } + /// A persisted snapshot: the tree structure with file contents replaced by content hashes. + /// The bytes themselves live in per-workspace `blobs/` files, so identical content is + /// stored once no matter how many snapshots reference it, and never base64-inflated. + private struct SnapshotManifest: Codable { + struct Node: Codable { + enum Kind: String, Codable { + case file, directory, symlink, missing + } + + var kind: Kind + var path: WorkspacePath + var permissions: POSIXPermissions? + var contentHash: String? + var symlinkTarget: String? + var children: [Node]? + } + + var version: Int + var id: UUID + var rootPath: WorkspacePath + var root: Node + } + func saveSnapshot(_ snapshot: Snapshot, workspaceId: UUID) async throws { try ensureWorkspaceDirectories(for: workspaceId) - try write(snapshot, to: snapshotURL(id: snapshot.id, workspaceId: workspaceId)) + let root = try writeManifestNode(snapshot.entry, workspaceId: workspaceId) + let manifest = SnapshotManifest( + version: 2, + id: snapshot.id, + rootPath: snapshot.rootPath, + root: root + ) + try write(manifest, to: snapshotURL(id: snapshot.id, workspaceId: workspaceId)) } func loadSnapshot(id: UUID, workspaceId: UUID) async throws -> Snapshot? { @@ -164,7 +209,97 @@ actor FileCheckpointStore: CheckpointStore { guard fileManager.fileExists(atPath: url.path) else { return nil } - return try read(Snapshot.self, from: url) + let data = try Data(contentsOf: url) + if let manifest = try? decoder.decode(SnapshotManifest.self, from: data) { + return Snapshot( + id: manifest.id, + rootPath: manifest.rootPath, + entry: try snapshotEntry(from: manifest.root, workspaceId: workspaceId) + ) + } + // Legacy format: the full snapshot tree with inline base64 contents. + return try decoder.decode(Snapshot.self, from: data) + } + + private func writeManifestNode( + _ entry: Snapshot.Entry, + workspaceId: UUID + ) throws -> SnapshotManifest.Node { + switch entry { + case let .missing(missing): + return SnapshotManifest.Node(kind: .missing, path: missing.path) + case let .file(file): + let hash = SHA256.hexDigest(of: file.data) + let url = blobURL(hash: hash, workspaceId: workspaceId) + if !fileManager.fileExists(atPath: url.path) { + try file.data.write(to: url, options: .atomic) + } + return SnapshotManifest.Node( + kind: .file, + path: file.path, + permissions: file.permissions, + contentHash: hash + ) + case let .symlink(symlink): + return SnapshotManifest.Node( + kind: .symlink, + path: symlink.path, + permissions: symlink.permissions, + symlinkTarget: symlink.target + ) + case let .directory(directory): + return SnapshotManifest.Node( + kind: .directory, + path: directory.path, + permissions: directory.permissions, + children: try directory.children.map { + try writeManifestNode($0, workspaceId: workspaceId) + } + ) + } + } + + private func snapshotEntry( + from node: SnapshotManifest.Node, + workspaceId: UUID + ) throws -> Snapshot.Entry { + switch node.kind { + case .missing: + return .missing(Snapshot.Missing(path: node.path)) + case .file: + guard let hash = node.contentHash else { + throw WorkspaceError.storageCorrupted("snapshot file node \(node.path) has no content hash") + } + let url = blobURL(hash: hash, workspaceId: workspaceId) + guard fileManager.fileExists(atPath: url.path) else { + throw WorkspaceError.storageCorrupted("missing content blob \(hash) for \(node.path)") + } + return .file( + Snapshot.File( + path: node.path, + data: try Data(contentsOf: url), + permissions: node.permissions ?? POSIXPermissions(0o644) + ) + ) + case .symlink: + return .symlink( + Snapshot.Symlink( + path: node.path, + target: node.symlinkTarget ?? "", + permissions: node.permissions ?? POSIXPermissions(0o777) + ) + ) + case .directory: + return .directory( + Snapshot.Directory( + path: node.path, + permissions: node.permissions ?? POSIXPermissions(0o755), + children: try (node.children ?? []).map { + try snapshotEntry(from: $0, workspaceId: workspaceId) + } + ) + ) + } } @discardableResult @@ -210,6 +345,30 @@ actor FileCheckpointStore: CheckpointStore { } } + func pruneMutationRecords(workspaceId: UUID, throughSequence: Int) async throws { + let jsonl = mutationsJsonlURL(workspaceId: workspaceId) + let legacy = legacyMutationsArrayURL(workspaceId: workspaceId) + guard fileManager.fileExists(atPath: jsonl.path) || fileManager.fileExists(atPath: legacy.path) else { + return + } + let lockURL = mutationsLockURL(workspaceId: workspaceId) + try Self.withMutationsExclusiveLock(at: lockURL) { + let records = try loadAllMutations(jsonl: jsonl, legacy: legacy) + .sorted { $0.sequence < $1.sequence } + guard let last = records.last else { + return + } + var kept = records.filter { $0.sequence > throughSequence } + if kept.isEmpty { + kept = [last] + } + guard kept.count != records.count else { + return + } + try writeAllMutationsAsJSONL(kept, to: jsonl) + } + } + private func loadAllMutations(jsonl: URL, legacy: URL) throws -> [MutationRecord] { if fileManager.fileExists(atPath: jsonl.path) { if fileManager.fileExists(atPath: legacy.path) { @@ -359,6 +518,7 @@ actor FileCheckpointStore: CheckpointStore { try fileManager.createDirectory(at: workspaceDirectoryURL(workspaceId: workspaceId), withIntermediateDirectories: true) try fileManager.createDirectory(at: checkpointsDirectoryURL(workspaceId: workspaceId), withIntermediateDirectories: true) try fileManager.createDirectory(at: snapshotsDirectoryURL(workspaceId: workspaceId), withIntermediateDirectories: true) + try fileManager.createDirectory(at: blobsDirectoryURL(workspaceId: workspaceId), withIntermediateDirectories: true) } private func workspaceDirectoryURL(workspaceId: UUID) -> URL { @@ -381,6 +541,14 @@ actor FileCheckpointStore: CheckpointStore { snapshotsDirectoryURL(workspaceId: workspaceId).appendingPathComponent("\(id.uuidString).json", isDirectory: false) } + private func blobsDirectoryURL(workspaceId: UUID) -> URL { + workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("blobs", isDirectory: true) + } + + private func blobURL(hash: String, workspaceId: UUID) -> URL { + blobsDirectoryURL(workspaceId: workspaceId).appendingPathComponent(hash, isDirectory: false) + } + private func mutationsJsonlURL(workspaceId: UUID) -> URL { workspaceDirectoryURL(workspaceId: workspaceId).appendingPathComponent("mutations.jsonl", isDirectory: false) } diff --git a/Sources/Workspace/FS/FileSystem.swift b/Sources/Workspace/FS/FileSystem.swift index f819a72..f097f42 100644 --- a/Sources/Workspace/FS/FileSystem.swift +++ b/Sources/Workspace/FS/FileSystem.swift @@ -33,6 +33,8 @@ public enum WorkspaceError: Error, CustomStringConvertible, Sendable { case mergeUncheckpointedChanges(parentWorkspaceId: UUID, baseMutationCursor: Int, currentMutationCursor: Int) /// A tracked workspace mutation could not be recorded. case mutationFailed(String) + /// Persisted checkpoint or snapshot storage is damaged (for example, a missing content blob). + case storageCorrupted(String) /// A human-readable description of the error. public var description: String { @@ -66,6 +68,8 @@ public enum WorkspaceError: Error, CustomStringConvertible, Sendable { return "cannot merge branch into workspace \(parentWorkspaceId.uuidString): the workspace has uncheckpointed changes (mutation sequence \(currentMutationCursor), branch base \(baseMutationCursor)); create a checkpoint or roll back before merging" case let .mutationFailed(message): return message + case let .storageCorrupted(message): + return "workspace storage is corrupted: \(message)" } } } diff --git a/Sources/Workspace/MutationRecord.swift b/Sources/Workspace/MutationRecord.swift index 5cd9e98..68a4e8d 100644 --- a/Sources/Workspace/MutationRecord.swift +++ b/Sources/Workspace/MutationRecord.swift @@ -15,6 +15,7 @@ struct MutationRecord: Sendable, Codable, Equatable { case applyEdits case applyReplacement case restoreSnapshot + case rollback case mergeWorkspace } diff --git a/Sources/Workspace/Support/SHA256.swift b/Sources/Workspace/Support/SHA256.swift new file mode 100644 index 0000000..7c26550 --- /dev/null +++ b/Sources/Workspace/Support/SHA256.swift @@ -0,0 +1,102 @@ +import Foundation + +/// A minimal, dependency-free SHA-256 implementation (FIPS 180-4). +/// +/// Used to content-address snapshot blobs in ``FileCheckpointStore``. Kept internal: it is not a +/// general-purpose cryptography API. +enum SHA256 { + /// Returns the lowercase hex digest of `data`. + static func hexDigest(of data: Data) -> String { + digest(of: data).map { String(format: "%02x", $0) }.joined() + } + + /// Returns the 32-byte SHA-256 digest of `data`. + static func digest(of data: Data) -> [UInt8] { + var h: [UInt32] = [ + 0x6a09_e667, 0xbb67_ae85, 0x3c6e_f372, 0xa54f_f53a, + 0x510e_527f, 0x9b05_688c, 0x1f83_d9ab, 0x5be0_cd19, + ] + + // Padding: append 0x80, zeros, then the 64-bit big-endian bit length. + var message = [UInt8](data) + let bitLength = UInt64(message.count) * 8 + message.append(0x80) + while message.count % 64 != 56 { + message.append(0) + } + for shift in stride(from: 56, through: 0, by: -8) { + message.append(UInt8(truncatingIfNeeded: bitLength >> UInt64(shift))) + } + + var w = [UInt32](repeating: 0, count: 64) + for blockStart in stride(from: 0, to: message.count, by: 64) { + for index in 0..<16 { + let offset = blockStart + index * 4 + w[index] = UInt32(message[offset]) << 24 + | UInt32(message[offset + 1]) << 16 + | UInt32(message[offset + 2]) << 8 + | UInt32(message[offset + 3]) + } + for index in 16..<64 { + let s0 = rotr(w[index - 15], 7) ^ rotr(w[index - 15], 18) ^ (w[index - 15] >> 3) + let s1 = rotr(w[index - 2], 17) ^ rotr(w[index - 2], 19) ^ (w[index - 2] >> 10) + w[index] = w[index - 16] &+ s0 &+ w[index - 7] &+ s1 + } + + var a = h[0], b = h[1], c = h[2], d = h[3] + var e = h[4], f = h[5], g = h[6], hh = h[7] + + for index in 0..<64 { + let s1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25) + let ch = (e & f) ^ (~e & g) + let temp1 = hh &+ s1 &+ ch &+ Self.k[index] &+ w[index] + let s0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22) + let maj = (a & b) ^ (a & c) ^ (b & c) + let temp2 = s0 &+ maj + + hh = g + g = f + f = e + e = d &+ temp1 + d = c + c = b + b = a + a = temp1 &+ temp2 + } + + h[0] &+= a + h[1] &+= b + h[2] &+= c + h[3] &+= d + h[4] &+= e + h[5] &+= f + h[6] &+= g + h[7] &+= hh + } + + var out: [UInt8] = [] + out.reserveCapacity(32) + for word in h { + out.append(UInt8(truncatingIfNeeded: word >> 24)) + out.append(UInt8(truncatingIfNeeded: word >> 16)) + out.append(UInt8(truncatingIfNeeded: word >> 8)) + out.append(UInt8(truncatingIfNeeded: word)) + } + return out + } + + private static func rotr(_ value: UInt32, _ amount: UInt32) -> UInt32 { + (value >> amount) | (value << (32 - amount)) + } + + private static let k: [UInt32] = [ + 0x428a_2f98, 0x7137_4491, 0xb5c0_fbcf, 0xe9b5_dba5, 0x3956_c25b, 0x59f1_11f1, 0x923f_82a4, 0xab1c_5ed5, + 0xd807_aa98, 0x1283_5b01, 0x2431_85be, 0x550c_7dc3, 0x72be_5d74, 0x80de_b1fe, 0x9bdc_06a7, 0xc19b_f174, + 0xe49b_69c1, 0xefbe_4786, 0x0fc1_9dc6, 0x240c_a1cc, 0x2de9_2c6f, 0x4a74_84aa, 0x5cb0_a9dc, 0x76f9_88da, + 0x983e_5152, 0xa831_c66d, 0xb003_27c8, 0xbf59_7fc7, 0xc6e0_0bf3, 0xd5a7_9147, 0x06ca_6351, 0x1429_2967, + 0x27b7_0a85, 0x2e1b_2138, 0x4d2c_6dfc, 0x5338_0d13, 0x650a_7354, 0x766a_0abb, 0x81c2_c92e, 0x9272_2c85, + 0xa2bf_e8a1, 0xa81a_664b, 0xc24b_8b70, 0xc76c_51a3, 0xd192_e819, 0xd699_0624, 0xf40e_3585, 0x106a_a070, + 0x19a4_c116, 0x1e37_6c08, 0x2748_774c, 0x34b0_bcb5, 0x391c_0cb3, 0x4ed8_aa4a, 0x5b9c_ca4f, 0x682e_6ff3, + 0x748f_82ee, 0x78a5_636f, 0x84c8_7814, 0x8cc7_0208, 0x90be_fffa, 0xa450_6ceb, 0xbef9_a3f7, 0xc671_78f2, + ] +} diff --git a/Sources/Workspace/Workspace+Checkpoints.swift b/Sources/Workspace/Workspace+Checkpoints.swift index def4594..0e65ac7 100644 --- a/Sources/Workspace/Workspace+Checkpoints.swift +++ b/Sources/Workspace/Workspace+Checkpoints.swift @@ -17,6 +17,9 @@ extension Workspace { } /// Restores the workspace to a prior checkpoint and records the rollback as a new checkpoint. + /// + /// The filesystem changes performed by the rollback are also recorded as a `rollback` + /// mutation, so the mutation log stays a complete account of tree changes. public func rollback(to checkpointId: UUID, label: String? = nil) async throws -> Checkpoint { try await ensureLoaded() try await reconcileCheckpointsWithStore() @@ -25,8 +28,20 @@ extension Workspace { id: checkpoint.snapshotId, workspaceId: checkpoint.workspaceId ) + let previousSnapshot = try await captureSnapshot() try await untrackedRestore(targetSnapshot) let restoredSnapshot = try await captureSnapshot() + + let delta = snapshotDelta(from: previousSnapshot.entry, to: restoredSnapshot.entry) + if delta.hasChanges { + try await appendMutation( + kind: .rollback, + touchedPaths: Array(delta.touchedPaths).sorted(), + fileChanges: delta.fileChanges, + diff: delta.fileChanges.count == 1 ? delta.fileChanges[0].diff : nil + ) + } + return try await persistCheckpoint( snapshot: restoredSnapshot, label: label, @@ -37,6 +52,16 @@ extension Workspace { ) } + /// Removes tracked mutation records with `sequence <= sequence` from the store, always + /// retaining the newest record so future sequence numbers stay monotonic. Long-running + /// workspaces can use this to keep the mutation log bounded. + public func pruneMutationHistory(throughSequence sequence: Int) async throws { + try await ensureLoaded() + try await store.pruneMutationRecords(workspaceId: workspaceId, throughSequence: sequence) + mutations = try await store.listMutationRecords(workspaceId: workspaceId) + nextMutationSequence = (mutations.map(\.sequence).max() ?? 0) + 1 + } + /// Lists checkpoints owned by this workspace. public func listCheckpoints() async throws -> [Checkpoint] { try await ensureLoaded() diff --git a/Tests/WorkspaceTests/CheckpointStoreTests.swift b/Tests/WorkspaceTests/CheckpointStoreTests.swift index e2cd2ec..1c30db9 100644 --- a/Tests/WorkspaceTests/CheckpointStoreTests.swift +++ b/Tests/WorkspaceTests/CheckpointStoreTests.swift @@ -483,6 +483,129 @@ struct CheckpointStoreTests { } } + @Test + func `FileCheckpointStore deduplicates snapshot content into blobs`() async throws { + let root = try makeTempDirectory() + defer { removeTempDirectory(root) } + + let store = FileCheckpointStore(rootDirectory: root) + let workspaceId = UUID() + + func fileEntry(_ path: WorkspacePath, _ text: String) -> Snapshot.Entry { + .file(Snapshot.File(path: path, data: Data(text.utf8), permissions: POSIXPermissions(0o644))) + } + func tree(_ children: [Snapshot.Entry]) -> Snapshot { + Snapshot( + rootPath: .root, + entry: .directory( + Snapshot.Directory(path: .root, permissions: .defaultDirectory, children: children) + ) + ) + } + + let first = tree([fileEntry("/a.txt", "alpha"), fileEntry("/b.txt", "beta")]) + let second = tree([fileEntry("/a.txt", "alpha"), fileEntry("/b.txt", "changed")]) + try await store.saveSnapshot(first, workspaceId: workspaceId) + try await store.saveSnapshot(second, workspaceId: workspaceId) + + // "alpha" is shared between the snapshots and stored once: three blobs, not four. + let blobsDirectory = root + .appendingPathComponent(workspaceId.uuidString, isDirectory: true) + .appendingPathComponent("blobs", isDirectory: true) + let blobs = try FileManager.default.contentsOfDirectory(atPath: blobsDirectory.path) + #expect(blobs.count == 3) + + // The manifest carries hashes, not base64-inflated contents. + let manifestURL = root + .appendingPathComponent(workspaceId.uuidString, isDirectory: true) + .appendingPathComponent("snapshots", isDirectory: true) + .appendingPathComponent("\(first.id.uuidString).json", isDirectory: false) + let manifestText = try String(contentsOf: manifestURL, encoding: .utf8) + #expect(!manifestText.contains(Data("alpha".utf8).base64EncodedString())) + #expect(manifestText.contains("\"version\"")) + #expect(manifestText.contains(SHA256.hexDigest(of: Data("alpha".utf8)))) + + // Loading reconstructs full snapshots, including contents and permissions. + #expect(try await store.loadSnapshot(id: first.id, workspaceId: workspaceId) == first) + #expect(try await store.loadSnapshot(id: second.id, workspaceId: workspaceId) == second) + + // A fresh store instance reads the same artifacts. + let reader = FileCheckpointStore(rootDirectory: root) + #expect(try await reader.loadSnapshot(id: second.id, workspaceId: workspaceId) == second) + } + + @Test + func `FileCheckpointStore loads legacy inline snapshots`() async throws { + let root = try makeTempDirectory() + defer { removeTempDirectory(root) } + + let workspaceId = UUID() + let snapshot = Snapshot( + rootPath: .root, + entry: .directory( + Snapshot.Directory( + path: .root, + permissions: .defaultDirectory, + children: [ + .file(Snapshot.File(path: "/x.txt", data: Data("legacy".utf8), permissions: POSIXPermissions(0o600))), + ] + ) + ) + ) + + // Write the legacy format (full snapshot with inline contents) directly. + let snapshotsDirectory = root + .appendingPathComponent(workspaceId.uuidString, isDirectory: true) + .appendingPathComponent("snapshots", isDirectory: true) + try FileManager.default.createDirectory(at: snapshotsDirectory, withIntermediateDirectories: true) + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(snapshot).write( + to: snapshotsDirectory.appendingPathComponent("\(snapshot.id.uuidString).json", isDirectory: false) + ) + + let store = FileCheckpointStore(rootDirectory: root) + #expect(try await store.loadSnapshot(id: snapshot.id, workspaceId: workspaceId) == snapshot) + } + + @Test + func `pruneMutationRecords bounds the log and keeps the newest record`() async throws { + let root = try makeTempDirectory() + defer { removeTempDirectory(root) } + + let store = FileCheckpointStore(rootDirectory: root) + let workspaceId = UUID() + for index in 0..<5 { + try await store.appendMutation( + MutationRecord( + sequence: 0, + workspaceId: workspaceId, + kind: .writeFile, + touchedPaths: [WorkspacePath(normalizing: "/file-\(index).txt")], + fileChanges: [] + ) + ) + } + + try await store.pruneMutationRecords(workspaceId: workspaceId, throughSequence: 3) + #expect(try await store.listMutationRecords(workspaceId: workspaceId).map(\.sequence) == [4, 5]) + + // Pruning everything still retains the newest record so sequences stay monotonic. + try await store.pruneMutationRecords(workspaceId: workspaceId, throughSequence: 100) + #expect(try await store.listMutationRecords(workspaceId: workspaceId).map(\.sequence) == [5]) + + let appended = try await store.appendMutation( + MutationRecord( + sequence: 0, + workspaceId: workspaceId, + kind: .writeFile, + touchedPaths: ["/next.txt"], + fileChanges: [] + ) + ) + #expect(appended.sequence == 6) + } + private func mutationsJsonlURL(root: URL, workspaceId: UUID) -> URL { root .appendingPathComponent(workspaceId.uuidString, isDirectory: true) diff --git a/Tests/WorkspaceTests/SHA256Tests.swift b/Tests/WorkspaceTests/SHA256Tests.swift new file mode 100644 index 0000000..654a818 --- /dev/null +++ b/Tests/WorkspaceTests/SHA256Tests.swift @@ -0,0 +1,40 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("SHA256") +struct SHA256Tests { + @Test + func `digest matches FIPS 180-4 test vectors`() { + #expect( + SHA256.hexDigest(of: Data()) + == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ) + #expect( + SHA256.hexDigest(of: Data("abc".utf8)) + == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ) + #expect( + SHA256.hexDigest(of: Data("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq".utf8)) + == "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" + ) + #expect( + SHA256.hexDigest(of: Data("hello world".utf8)) + == "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + ) + } + + @Test + func `digest handles multi-block and block-boundary inputs`() { + // 55, 56, and 64 bytes cross the padding boundaries; 1000 bytes spans many blocks. + let a55 = Data(String(repeating: "a", count: 55).utf8) + let a56 = Data(String(repeating: "a", count: 56).utf8) + let a64 = Data(String(repeating: "a", count: 64).utf8) + let a1000 = Data(String(repeating: "a", count: 1000).utf8) + + #expect(SHA256.hexDigest(of: a55) == "9f4390f8d30c2dd92ec9f095b65e2b9ae9b0a925a5258e241c9f1e910f734318") + #expect(SHA256.hexDigest(of: a56) == "b35439a4ac6f0948b6d6f9e3c6af0f5f590ce20f1bde7090ef7970686ec6738a") + #expect(SHA256.hexDigest(of: a64) == "ffe054fe7ae0cb6dc65c3af9b61d5209f439851db43d0ba5997337df154668eb") + #expect(SHA256.hexDigest(of: a1000) == "41edece42d63e8d9bf515a9ba6932e1c20cbc9f5a5d134645adb5db1b9737ea3") + } +} diff --git a/Tests/WorkspaceTests/WorkspaceCheckpointTests.swift b/Tests/WorkspaceTests/WorkspaceCheckpointTests.swift index 2934ef2..c9efb3d 100644 --- a/Tests/WorkspaceTests/WorkspaceCheckpointTests.swift +++ b/Tests/WorkspaceTests/WorkspaceCheckpointTests.swift @@ -82,6 +82,10 @@ private actor FlakySnapshotCheckpointStore: CheckpointStore { func listMutationRecords(workspaceId: UUID) async throws -> [MutationRecord] { try await base.listMutationRecords(workspaceId: workspaceId) } + + func pruneMutationRecords(workspaceId: UUID, throughSequence: Int) async throws { + try await base.pruneMutationRecords(workspaceId: workspaceId, throughSequence: throughSequence) + } } @Suite("Workspace checkpoints") @@ -135,6 +139,42 @@ struct WorkspaceCheckpointTests { #expect(events[0].checkpoint == rollbackCheckpoint) } + @Test + func `rollback appends a rollback mutation record`() async throws { + let workspace = Workspace(filesystem: InMemoryFilesystem()) + try await workspace.writeFile("/file.txt", content: "one") + let checkpoint = try await workspace.createCheckpoint(label: "v1") + try await workspace.writeFile("/file.txt", content: "two") + + _ = try await workspace.rollback(to: checkpoint.id) + + let mutations = try await workspace.mutationRecords() + #expect(mutations.map(\.kind) == [.writeFile, .writeFile, .rollback]) + #expect(mutations.last?.touchedPaths == ["/file.txt"]) + #expect(mutations.last?.fileChanges.first?.diff?.hunks.isEmpty == false) + #expect(try await workspace.readFile("/file.txt") == "one") + } + + @Test + func `pruneMutationHistory bounds the log while keeping sequences monotonic`() async throws { + let root = try makeTempDirectory() + defer { removeTempDirectory(root) } + + let workspace = Workspace(filesystem: InMemoryFilesystem(), storage: .directory(at: root)) + for index in 0..<5 { + try await workspace.writeFile("/file.txt", content: "revision \(index)") + } + + try await workspace.pruneMutationHistory(throughSequence: 3) + #expect(try await workspace.mutationRecords().map(\.sequence) == [4, 5]) + + try await workspace.pruneMutationHistory(throughSequence: 100) + #expect(try await workspace.mutationRecords().map(\.sequence) == [5]) + + try await workspace.writeFile("/file.txt", content: "after prune") + #expect(try await workspace.mutationRecords().map(\.sequence) == [5, 6]) + } + @Test func `branch writes are isolated and merge creates one merge checkpoint`() async throws { let workspace = Workspace(filesystem: InMemoryFilesystem()) diff --git a/Tests/WorkspaceTests/WorkspaceInternalsTests.swift b/Tests/WorkspaceTests/WorkspaceInternalsTests.swift index 79b3d87..78f39b7 100644 --- a/Tests/WorkspaceTests/WorkspaceInternalsTests.swift +++ b/Tests/WorkspaceTests/WorkspaceInternalsTests.swift @@ -47,6 +47,8 @@ private actor CountingCheckpointStore: CheckpointStore { return mutations.filter { $0.workspaceId == workspaceId } } + func pruneMutationRecords(workspaceId: UUID, throughSequence: Int) async throws {} + func listCounts() -> (checkpoints: Int, mutations: Int) { (checkpointListCount, mutationListCount) } From 028800ace65dd9f1e6b0e0fdcb36ea05519b5835 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 04:04:37 +0000 Subject: [PATCH 6/7] Add configurable tracking policy, public mutation history, cheaper planning Workspace gains a TrackingPolicy controlling how much detail tracked writes record: - .full (default) keeps today's behavior: full mutation records with per-file line diffs. - .fullDiffs(maxDiffBytes:) caps the content size for which diffs are computed, so huge files no longer trigger whole-content diffing. - .pathsOnly records touched paths and effects without diffs. - .disabled records no history; change events still fire, and the policy is inherited by branches. Merge's uncheckpointed-changes guard cannot see edits made in this mode (documented). MutationRecord and mutationRecords() are now public, so consumers can actually read the history the workspace records. Event planning for deletions and moves now walks structure only (stats and listings) instead of capturing full file contents, removing a whole-subtree content read from every removeItem/moveItem/copyItem. Replacement change events are driven by replacement counts instead of diff presence so they keep firing when diffs are suppressed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01G29SXqzHqDycqXGa4dRr4x --- README.md | 15 +- Sources/Workspace/MutationRecord.swift | 29 ++-- Sources/Workspace/Workspace+Branching.swift | 3 +- Sources/Workspace/Workspace+Internals.swift | 31 ++++- Sources/Workspace/Workspace.swift | 128 +++++++++++++++--- .../WorkspaceTests/TrackingPolicyTests.swift | 104 ++++++++++++++ 6 files changed, 275 insertions(+), 35 deletions(-) create mode 100644 Tests/WorkspaceTests/TrackingPolicyTests.swift diff --git a/README.md b/README.md index e44e1a5..c160767 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,20 @@ print(config.enabled) // true ### Tracked Writes -Every public write records an internal mutation: +Every public write records a mutation. The history is public — `mutationRecords()` returns the ordered records including per-file effects and text diffs — and the amount of detail is configurable per workspace: + +```swift +let workspace = Workspace( + filesystem: InMemoryFilesystem(), + tracking: .fullDiffs(maxDiffBytes: 1_000_000) // cap diff computation to 1 MB files +) +// .full — full diffs, no size cap (default) +// .pathsOnly — touched paths and effects, no text diffs +// .disabled — no history at all; change events still fire, but merge cannot +// detect uncheckpointed edits made in this mode +``` + +Tracked writes: ```swift try await workspace.writeFile("/notes/todo.txt", content: "one") diff --git a/Sources/Workspace/MutationRecord.swift b/Sources/Workspace/MutationRecord.swift index 68a4e8d..80c6c40 100644 --- a/Sources/Workspace/MutationRecord.swift +++ b/Sources/Workspace/MutationRecord.swift @@ -1,9 +1,13 @@ import Foundation /// A recorded filesystem mutation emitted by ``Workspace``. -struct MutationRecord: Sendable, Codable, Equatable { +/// +/// Every tracked write appends one record (unless the workspace's ``Workspace/TrackingPolicy`` +/// disables history). Records are ordered by `sequence`, which the checkpoint store assigns +/// monotonically per workspace. +public struct MutationRecord: Sendable, Codable, Equatable { /// The coarse operation kind for filtering and tooling. - enum Kind: String, Sendable, Codable { + public enum Kind: String, Sendable, Codable { case writeFile case appendFile case writeData @@ -19,13 +23,20 @@ struct MutationRecord: Sendable, Codable, Equatable { case mergeWorkspace } - var sequence: Int - var workspaceId: UUID - var timestamp: Date - var kind: Kind - var touchedPaths: [WorkspacePath] - var fileChanges: [FileEdit.FileChange] - var diff: TextDiff? + /// The store-assigned, per-workspace monotonic sequence number. + public internal(set) var sequence: Int + /// The workspace that performed the mutation. + public var workspaceId: UUID + /// When the mutation was recorded. + public var timestamp: Date + /// The coarse operation kind. + public var kind: Kind + /// The canonicalized paths the mutation touched. + public var touchedPaths: [WorkspacePath] + /// Per-file effects, including text diffs when the tracking policy records them. + public var fileChanges: [FileEdit.FileChange] + /// A convenience diff populated when the mutation changed exactly one text file. + public var diff: TextDiff? init( sequence: Int, diff --git a/Sources/Workspace/Workspace+Branching.swift b/Sources/Workspace/Workspace+Branching.swift index 0359768..a5623ba 100644 --- a/Sources/Workspace/Workspace+Branching.swift +++ b/Sources/Workspace/Workspace+Branching.swift @@ -26,7 +26,8 @@ extension Workspace { filesystem: branchFilesystem, store: store, baseCheckpointId: baseCheckpointId, - baseMutationCursor: latestMutationSequence() + baseMutationCursor: latestMutationSequence(), + tracking: tracking ) _ = try await branch.seedBranchCheckpoint( snapshot: snapshot, diff --git a/Sources/Workspace/Workspace+Internals.swift b/Sources/Workspace/Workspace+Internals.swift index bb99eb6..1d96bed 100644 --- a/Sources/Workspace/Workspace+Internals.swift +++ b/Sources/Workspace/Workspace+Internals.swift @@ -94,6 +94,11 @@ extension Workspace { edit: FileEdit, apply: () async throws -> Void ) async throws { + if case .disabled = tracking { + try await apply() + return + } + try await ensureLoaded() let preview = try await previewEdits([edit]) try await apply() @@ -108,6 +113,11 @@ extension Workspace { } func performBinaryWrite(data: Data, path: WorkspacePath) async throws { + if case .disabled = tracking { + try await untrackedWriteData(data, to: path) + return + } + let effect: FileEdit.Effect let kind: FileTree.Kind @@ -117,7 +127,11 @@ extension Workspace { throw WorkspaceError.unsupported("cannot write data to a directory: \(path)") } kind = info.kind - effect = try await readData(from: path) == data ? .unchanged : .modified + if computesDiffs { + effect = try await readData(from: path) == data ? .unchanged : .modified + } else { + effect = .modified + } } else { kind = .file effect = .created @@ -200,6 +214,9 @@ extension Workspace { fileChanges: [FileEdit.FileChange], diff: TextDiff? ) async throws { + if case .disabled = tracking { + return + } let provisional = MutationRecord( sequence: 0, workspaceId: workspaceId, @@ -218,7 +235,11 @@ extension Workspace { mutations.map(\.sequence).max() ?? 0 } - func mutationRecords() async throws -> [MutationRecord] { + /// Returns the tracked mutation history for this workspace, oldest first. + /// + /// The amount of detail per record is governed by the workspace's ``Workspace/TrackingPolicy``; + /// with ``Workspace/TrackingPolicy/disabled`` the history is empty. + public func mutationRecords() async throws -> [MutationRecord] { try await ensureLoaded() return mutations } @@ -461,6 +482,12 @@ extension Workspace { } func utf8TextFileDiff(oldData: Data, newData: Data) -> TextDiff? { + guard case let .fullDiffs(maxDiffBytes) = tracking else { + return nil + } + if let maxDiffBytes, oldData.count > maxDiffBytes || newData.count > maxDiffBytes { + return nil + } guard let oldString = String(data: oldData, encoding: .utf8), let newString = String(data: newData, encoding: .utf8) else { diff --git a/Sources/Workspace/Workspace.swift b/Sources/Workspace/Workspace.swift index c3e072d..4ddfe9c 100644 --- a/Sources/Workspace/Workspace.swift +++ b/Sources/Workspace/Workspace.swift @@ -17,6 +17,21 @@ public actor Workspace { case directory(at: URL) } + /// Controls how much detail tracked writes record in the mutation log. + public enum TrackingPolicy: Sendable, Equatable { + /// Record full mutation records including per-file line diffs. `maxDiffBytes` caps the + /// content size for which diffs are computed; larger files are recorded without a diff. + case fullDiffs(maxDiffBytes: Int?) + /// Record mutation records with touched paths and effects but no text diffs. + case pathsOnly + /// Record no mutation history at all. Change events still fire, but ``Workspace/merge(_:label:)`` + /// cannot detect uncheckpointed edits made in this mode. + case disabled + + /// Full diffs with no size cap. The default. + public static let full = TrackingPolicy.fullDiffs(maxDiffBytes: nil) + } + struct CheckpointWatcher { var deliveredCheckpointIds: Set var continuation: AsyncStream.Continuation @@ -29,6 +44,8 @@ public actor Workspace { /// The property is immutable after initialization, but the returned filesystem retains its normal /// ``FileSystem`` capabilities. public nonisolated let filesystem: any FileSystem + /// How much detail tracked writes record. See ``TrackingPolicy``. + public nonisolated let tracking: TrackingPolicy let store: any CheckpointStore let baseCheckpointId: UUID? /// The parent's mutation sequence at the moment this workspace was branched. Merge uses it to @@ -66,13 +83,15 @@ public actor Workspace { public init( workspaceId: UUID = UUID(), filesystem: any FileSystem, - storage: Storage = .inMemory + storage: Storage = .inMemory, + tracking: TrackingPolicy = .full ) { self.init( workspaceId: workspaceId, filesystem: filesystem, store: Self.makeStore(for: storage), - baseCheckpointId: nil + baseCheckpointId: nil, + tracking: tracking ) } @@ -81,13 +100,15 @@ public actor Workspace { filesystem: any FileSystem, store: any CheckpointStore, baseCheckpointId: UUID? = nil, - baseMutationCursor: Int? = nil + baseMutationCursor: Int? = nil, + tracking: TrackingPolicy = .full ) { self.workspaceId = workspaceId self.filesystem = filesystem self.store = store self.baseCheckpointId = baseCheckpointId self.baseMutationCursor = baseMutationCursor + self.tracking = tracking } static func makeStore(for storage: Storage) -> any CheckpointStore { @@ -796,6 +817,7 @@ public actor Workspace { path: path, replacements: replacement.count, diff: textDiff(from: originalContent, to: replacement.updatedContent) + ?? TextDiff(hunks: []) ), updatedContent: replacement.updatedContent, nodeKind: info.kind @@ -921,9 +943,16 @@ public actor Workspace { ) async throws -> FileEdit.Entry { switch edit { case let .writeFile(path, content): - let original = try await readUTF8IfPresent(path, on: target) let info = try await statIfPresent(path, on: target) - let effect = effect(forOriginalContent: original, updatedContent: content) + let original = computesDiffs ? try await readUTF8IfPresent(path, on: target) : nil + let effect: FileEdit.Effect = + if info == nil { + .created + } else if computesDiffs { + effect(forOriginalContent: original, updatedContent: content) + } else { + .modified + } return FileEdit.Entry( edit: edit, changeState: changeState(for: effect), @@ -939,10 +968,17 @@ public actor Workspace { ] ) case let .appendFile(path, content): - let original = try await readUTF8IfPresent(path, on: target) - let updated = (original ?? "") + content let info = try await statIfPresent(path, on: target) - let effect = effect(forOriginalContent: original, updatedContent: updated) + let original = computesDiffs ? try await readUTF8IfPresent(path, on: target) : nil + let updated = (original ?? "") + content + let effect: FileEdit.Effect = + if info == nil { + .created + } else if computesDiffs { + effect(forOriginalContent: original, updatedContent: updated) + } else { + .modified + } return FileEdit.Entry( edit: edit, changeState: changeState(for: effect), @@ -1033,7 +1069,7 @@ public actor Workspace { } let diff: TextDiff? - if info.kind == .symlink { + if info.kind == .symlink || !computesDiffs { diff = nil } else if let originalContent = try await readUTF8IfPresent( path, @@ -1194,11 +1230,43 @@ public actor Workspace { ] } + /// A content-free tree capture used only to plan change events. Unlike ``RollbackCapture`` + /// it never reads file bytes, so planning deletion or transfer events for a large subtree + /// costs stats and listings, not file contents. + private indirect enum StructureCapture: Sendable { + case missing + case file(path: WorkspacePath) + case symlink(path: WorkspacePath) + case directory(path: WorkspacePath, children: [StructureCapture]) + } + + private func structureCapture(path: WorkspacePath, on target: any FileSystem) async throws -> StructureCapture { + guard await target.exists(path: path) else { + return .missing + } + + let info = try await target.stat(path: path) + switch info.kind { + case .directory: + let entries = try await target.listDirectory(path: path) + let children = try await entries + .sorted { $0.name < $1.name } + .asyncMap { [self] entry in + try await self.structureCapture(path: path.appending(entry.name), on: target) + } + return .directory(path: path, children: children) + case .symlink: + return .symlink(path: path) + case .file: + return .file(path: path) + } + } + private func plannedDeletionEvents( at path: WorkspacePath, on target: any FileSystem ) async throws -> [ChangeEvent] { - let capture = try await rollbackCapture(path: path, on: target) + let capture = try await structureCapture(path: path, on: target) return flattenDeletionEvents(from: capture) } @@ -1208,7 +1276,7 @@ public actor Workspace { kind: ChangeEvent.Kind, on target: any FileSystem ) async throws -> [ChangeEvent] { - let capture = try await rollbackCapture(path: sourcePath, on: target) + let capture = try await structureCapture(path: sourcePath, on: target) return flattenTransferEvents( from: capture, sourceRoot: sourcePath, @@ -1218,7 +1286,7 @@ public actor Workspace { } private func changeEvents(for replacement: PlannedReplacement) -> [ChangeEvent] { - guard !replacement.change.diff.hunks.isEmpty else { + guard replacement.change.replacements > 0 else { return [] } @@ -1231,15 +1299,15 @@ public actor Workspace { ] } - private func flattenDeletionEvents(from capture: RollbackCapture) -> [ChangeEvent] { + private func flattenDeletionEvents(from capture: StructureCapture) -> [ChangeEvent] { switch capture { case .missing: return [] - case let .file(path, _, _): + case let .file(path): return [ChangeEvent(kind: .deleted, path: path, nodeKind: .file)] - case let .symlink(path, _, _): + case let .symlink(path): return [ChangeEvent(kind: .deleted, path: path, nodeKind: .symlink)] - case let .directory(path, _, children): + case let .directory(path, children): var events: [ChangeEvent] = [] for child in children { events += flattenDeletionEvents(from: child) @@ -1256,7 +1324,7 @@ public actor Workspace { } private func flattenTransferEvents( - from capture: RollbackCapture, + from capture: StructureCapture, sourceRoot: WorkspacePath, destinationRoot: WorkspacePath, kind: ChangeEvent.Kind @@ -1264,7 +1332,7 @@ public actor Workspace { switch capture { case .missing: return [] - case let .file(path, _, _): + case let .file(path): return [ ChangeEvent( kind: kind, @@ -1273,7 +1341,7 @@ public actor Workspace { nodeKind: .file ), ] - case let .symlink(path, _, _): + case let .symlink(path): return [ ChangeEvent( kind: kind, @@ -1282,7 +1350,7 @@ public actor Workspace { nodeKind: .symlink ), ] - case let .directory(path, _, children): + case let .directory(path, children): var events: [ChangeEvent] = [ ChangeEvent( kind: kind, @@ -1350,8 +1418,24 @@ public actor Workspace { } } - private func textDiff(from originalContent: String, to updatedContent: String) -> TextDiff { - TextDiff.lineBased(from: originalContent, to: updatedContent) + /// Whether the tracking policy calls for line diffs at all. + var computesDiffs: Bool { + if case .fullDiffs = tracking { + return true + } + return false + } + + private func textDiff(from originalContent: String, to updatedContent: String) -> TextDiff? { + guard case let .fullDiffs(maxDiffBytes) = tracking else { + return nil + } + if let maxDiffBytes, + originalContent.utf8.count > maxDiffBytes || updatedContent.utf8.count > maxDiffBytes + { + return nil + } + return TextDiff.lineBased(from: originalContent, to: updatedContent) } private func describe(_ error: any Error) -> String { diff --git a/Tests/WorkspaceTests/TrackingPolicyTests.swift b/Tests/WorkspaceTests/TrackingPolicyTests.swift new file mode 100644 index 0000000..da2b179 --- /dev/null +++ b/Tests/WorkspaceTests/TrackingPolicyTests.swift @@ -0,0 +1,104 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Tracking policy") +struct TrackingPolicyTests { + @Test + func `full tracking exposes public mutation history with diffs`() async throws { + let workspace = Workspace(filesystem: InMemoryFilesystem()) + try await workspace.writeFile("/note.txt", content: "one\n") + try await workspace.writeFile("/note.txt", content: "two\n") + + let mutations = try await workspace.mutationRecords() + #expect(mutations.map(\.kind) == [.writeFile, .writeFile]) + #expect(mutations.map(\.sequence) == [1, 2]) + #expect(mutations[1].diff?.hunks.isEmpty == false) + #expect(mutations[1].touchedPaths == ["/note.txt"]) + } + + @Test + func `pathsOnly tracking records effects without diffs`() async throws { + let workspace = Workspace(filesystem: InMemoryFilesystem(), tracking: .pathsOnly) + try await workspace.writeFile("/note.txt", content: "one") + try await workspace.writeFile("/note.txt", content: "two") + try await workspace.removeItem(at: "/note.txt") + + let mutations = try await workspace.mutationRecords() + #expect(mutations.map(\.kind) == [.writeFile, .writeFile, .removeItem]) + #expect(mutations.allSatisfy { $0.diff == nil }) + #expect(mutations.allSatisfy { record in record.fileChanges.allSatisfy { $0.diff == nil } }) + #expect(mutations[1].fileChanges.first?.effect == .modified) + #expect(mutations[1].touchedPaths == ["/note.txt"]) + #expect(mutations[2].fileChanges.first?.effect == .deleted) + } + + @Test + func `pathsOnly tracking keeps replacement counts and change events`() async throws { + let workspace = Workspace(filesystem: InMemoryFilesystem(), tracking: .pathsOnly) + try await workspace.writeFile("/a.txt", content: "old old") + + let stream = await workspace.watchChanges(at: "/a.txt", recursive: false) + var iterator = stream.makeAsyncIterator() + + let result = try await workspace.applyReplacement( + ReplacementRequest(pattern: "/a.txt", search: "old", replacement: "new") + ) + + #expect(result.changes.first?.replacements == 2) + #expect(result.changes.first?.diff.hunks.isEmpty == true) + #expect(try await workspace.readFile("/a.txt") == "new new") + + let event = await iterator.next() + #expect(event?.kind == .modified) + #expect(event?.path == "/a.txt") + } + + @Test + func `disabled tracking records nothing but still emits change events`() async throws { + let workspace = Workspace(filesystem: InMemoryFilesystem(), tracking: .disabled) + + let stream = await workspace.watchChanges(at: "/", recursive: true) + var iterator = stream.makeAsyncIterator() + + try await workspace.writeFile("/note.txt", content: "one") + try await workspace.appendFile("/note.txt", content: " two") + try await workspace.writeData(Data([0xFF]), to: "/bin.dat") + + #expect(try await workspace.readFile("/note.txt") == "one two") + #expect(try await workspace.mutationRecords().isEmpty) + + let event = await iterator.next() + #expect(event?.kind == .created) + #expect(event?.path == "/note.txt") + } + + @Test + func `diff size cap suppresses diffs for large contents only`() async throws { + let workspace = Workspace( + filesystem: InMemoryFilesystem(), + tracking: .fullDiffs(maxDiffBytes: 16) + ) + try await workspace.writeFile("/small.txt", content: "tiny\n") + try await workspace.writeFile("/big.txt", content: String(repeating: "x", count: 64)) + + let mutations = try await workspace.mutationRecords() + #expect(mutations[0].fileChanges.first?.diff != nil) + #expect(mutations[1].fileChanges.first?.diff == nil) + #expect(mutations[1].fileChanges.first?.effect == .created) + } + + @Test + func `branches inherit the parent tracking policy`() async throws { + let workspace = Workspace(filesystem: InMemoryFilesystem(), tracking: .pathsOnly) + try await workspace.writeFile("/base.txt", content: "base") + _ = try await workspace.createCheckpoint(label: "base") + + let branch = try await workspace.branch() + #expect(branch.tracking == .pathsOnly) + + try await branch.writeFile("/base.txt", content: "branch") + let mutations = try await branch.mutationRecords() + #expect(mutations.allSatisfy { record in record.fileChanges.allSatisfy { $0.diff == nil } }) + } +} From d70b0252048b440ace6fd34691531643c9fc0ead Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 9 Jul 2026 04:07:29 +0000 Subject: [PATCH 7/7] Add three-way merge with structured conflicts mergeThreeWay(_:label:) resolves each path against the branch's base checkpoint snapshot: the side that changed relative to the base wins, identical changes on both sides merge cleanly, and branch deletions propagate. Unlike merge(_:), the parent may have advanced -- with or without checkpoints -- since the branch was created. When both sides changed the same path differently, nothing is applied and the result carries structured MergeConflict values (bothModified, modifiedAndDeleted, bothCreated) with base-to-ours and base-to-theirs text diffs, so callers can present or resolve conflicts instead of handling a thrown error. A clean merge restores the merged tree, records a mergeWorkspace mutation, and creates a checkpoint carrying the mergedFrom lineage fields. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01G29SXqzHqDycqXGa4dRr4x --- README.md | 14 ++ Sources/Workspace/Workspace+Branching.swift | 237 ++++++++++++++++++ Tests/WorkspaceTests/ThreeWayMergeTests.swift | 144 +++++++++++ 3 files changed, 395 insertions(+) create mode 100644 Tests/WorkspaceTests/ThreeWayMergeTests.swift diff --git a/README.md b/README.md index c160767..8e499da 100644 --- a/README.md +++ b/README.md @@ -252,6 +252,20 @@ print(merged.mergedFromCheckpointId == branchHead.id) // true `merge(_:)` is optimistic. If the parent workspace head changed after `branch()` was created, merge throws `WorkspaceError.mergeConflict(parentWorkspaceId:expectedBase:actualHead:)`. If the parent made tracked writes after `branch()` without creating a checkpoint, merge throws `WorkspaceError.mergeUncheckpointedChanges(parentWorkspaceId:baseMutationCursor:currentMutationCursor:)` instead of silently overwriting those edits; create a checkpoint or roll the parent back first. +`mergeThreeWay(_:label:)` merges even when both sides advanced. Each path is resolved against the branch's base checkpoint: the side that changed wins, identical changes merge cleanly, and when both sides changed the same path differently the merge applies nothing and returns structured conflicts instead of throwing: + +```swift +let result = try await workspace.mergeThreeWay(branch, label: "merge draft") +if result.applied { + print("merged as checkpoint \(result.checkpoint!.id)") +} else { + for conflict in result.conflicts { + print("conflict at \(conflict.path): \(conflict.kind)") + // conflict.oursDiff / conflict.theirsDiff carry base→side diffs for text files + } +} +``` + ## Common Filesystem Patterns ### Rooted Disk Workspace diff --git a/Sources/Workspace/Workspace+Branching.swift b/Sources/Workspace/Workspace+Branching.swift index a5623ba..6c3d607 100644 --- a/Sources/Workspace/Workspace+Branching.swift +++ b/Sources/Workspace/Workspace+Branching.swift @@ -101,6 +101,243 @@ extension Workspace { ) } + /// A path where the workspace and a branch changed the same entry in incompatible ways + /// relative to their merge base. + public struct MergeConflict: Sendable, Codable, Equatable { + /// How the two sides disagree. + public enum Kind: String, Sendable, Codable { + /// Both sides modified the entry with different results. + case bothModified + /// One side modified the entry and the other deleted it. + case modifiedAndDeleted + /// Both sides created different entries at the same path. + case bothCreated + } + + /// The conflicting path. + public var path: WorkspacePath + /// The conflict classification. + public var kind: Kind + /// Diff from the base content to this workspace's content, when both sides are UTF-8 text. + public var oursDiff: TextDiff? + /// Diff from the base content to the branch's content, when both sides are UTF-8 text. + public var theirsDiff: TextDiff? + } + + /// The outcome of ``Workspace/mergeThreeWay(_:label:)``. + public struct ThreeWayMergeResult: Sendable { + /// The merge checkpoint, present when the merge applied cleanly. + public let checkpoint: Checkpoint? + /// The conflicts that prevented the merge; empty when `checkpoint` is present. + public let conflicts: [MergeConflict] + /// Whether the merge was applied to this workspace. + public var applied: Bool { checkpoint != nil } + } + + /// Merges a branch using per-path three-way resolution against the branch's base checkpoint. + /// + /// For every path, the side that changed relative to the base wins; identical changes on + /// both sides merge cleanly. When both sides changed a path differently the merge applies + /// nothing and returns the conflicts instead. Unlike ``merge(_:label:)``, this workspace may + /// have advanced (checkpointed or not) since the branch was created. + public func mergeThreeWay(_ other: Workspace, label: String? = nil) async throws -> ThreeWayMergeResult { + try await ensureLoaded() + try await reconcileCheckpointsWithStore() + + guard let baseCheckpointId = await other.mergeBaseCheckpointId() else { + throw WorkspaceError.unsupported( + "three-way merge requires a branch created from a checkpointed workspace" + ) + } + let baseCheckpoint = try checkpointOrThrow(id: baseCheckpointId) + let baseSnapshot = try await loadSnapshotOrThrow( + id: baseCheckpoint.snapshotId, + workspaceId: baseCheckpoint.workspaceId + ) + + let oursSnapshot = try await captureSnapshot() + let theirsSnapshot = try await other.captureSnapshot() + let incomingHead = try await other.currentHeadCheckpointId() + + var base: [WorkspacePath: MergeNode] = [:] + var ours: [WorkspacePath: MergeNode] = [:] + var theirs: [WorkspacePath: MergeNode] = [:] + Self.flattenMergeNodes(baseSnapshot.entry, into: &base) + Self.flattenMergeNodes(oursSnapshot.entry, into: &ours) + Self.flattenMergeNodes(theirsSnapshot.entry, into: &theirs) + base.removeValue(forKey: .root) + ours.removeValue(forKey: .root) + theirs.removeValue(forKey: .root) + + var merged = ours + var conflicts: [MergeConflict] = [] + let allPaths = Set(base.keys).union(ours.keys).union(theirs.keys).sorted() + + for path in allPaths { + let baseNode = base[path] + let oursNode = ours[path] + let theirsNode = theirs[path] + + if theirsNode == baseNode || oursNode == theirsNode { + continue // theirs unchanged, or both sides agree: keep ours. + } + if oursNode == baseNode { + // Only theirs changed: take it. + if let theirsNode { + merged[path] = theirsNode + } else { + merged.removeValue(forKey: path) + } + continue + } + + let kind: MergeConflict.Kind + if baseNode == nil { + kind = .bothCreated + } else if oursNode == nil || theirsNode == nil { + kind = .modifiedAndDeleted + } else { + kind = .bothModified + } + conflicts.append( + MergeConflict( + path: path, + kind: kind, + oursDiff: Self.conflictDiff(from: baseNode, to: oursNode), + theirsDiff: Self.conflictDiff(from: baseNode, to: theirsNode) + ) + ) + } + + guard conflicts.isEmpty else { + return ThreeWayMergeResult(checkpoint: nil, conflicts: conflicts) + } + + let rootPermissions: POSIXPermissions = + if case let .directory(directory) = oursSnapshot.entry { + directory.permissions + } else { + .defaultDirectory + } + let mergedSnapshot = Snapshot( + rootPath: .root, + entry: Self.buildEntryTree(from: merged, rootPermissions: rootPermissions) + ) + + try await untrackedRestore(mergedSnapshot) + + let delta = snapshotDelta(from: oursSnapshot.entry, to: mergedSnapshot.entry) + if delta.hasChanges { + try await appendMutation( + kind: .mergeWorkspace, + touchedPaths: Array(delta.touchedPaths).sorted(), + fileChanges: delta.fileChanges, + diff: delta.fileChanges.count == 1 ? delta.fileChanges[0].diff : nil + ) + } + + let checkpoint = try await persistCheckpoint( + snapshot: try await captureSnapshot(), + label: label, + parentCheckpointId: headCheckpointId, + baseCheckpointId: self.baseCheckpointId, + mergedFromWorkspaceId: other.workspaceId, + mergedFromCheckpointId: incomingHead, + rollbackSourceCheckpointId: nil, + eventKind: .merged + ) + return ThreeWayMergeResult(checkpoint: checkpoint, conflicts: []) + } + + /// A comparable flat view of one snapshot entry, used for three-way resolution. + struct MergeNode: Equatable { + var kind: FileTree.Kind + var permissions: POSIXPermissions + var fileData: Data? + var symlinkTarget: String? + } + + static func flattenMergeNodes(_ entry: Snapshot.Entry, into nodes: inout [WorkspacePath: MergeNode]) { + switch entry { + case .missing: + return + case let .file(file): + nodes[file.path] = MergeNode(kind: .file, permissions: file.permissions, fileData: file.data) + case let .symlink(symlink): + nodes[symlink.path] = MergeNode( + kind: .symlink, + permissions: symlink.permissions, + symlinkTarget: symlink.target + ) + case let .directory(directory): + nodes[directory.path] = MergeNode(kind: .directory, permissions: directory.permissions) + for child in directory.children { + flattenMergeNodes(child, into: &nodes) + } + } + } + + private static func conflictDiff(from base: MergeNode?, to side: MergeNode?) -> TextDiff? { + let baseData = base.map { $0.fileData ?? Data() } ?? Data() + let sideData = side.map { $0.fileData ?? Data() } ?? Data() + guard let baseText = String(data: baseData, encoding: .utf8), + let sideText = String(data: sideData, encoding: .utf8) + else { + return nil + } + let diff = TextDiff.lineBased(from: baseText, to: sideText) + return diff.hunks.isEmpty ? nil : diff + } + + /// Rebuilds a snapshot tree from a flat path-to-node map. + static func buildEntryTree( + from nodes: [WorkspacePath: MergeNode], + rootPermissions: POSIXPermissions + ) -> Snapshot.Entry { + var childNames: [WorkspacePath: [String]] = [:] + for path in nodes.keys { + childNames[path.dirname, default: []].append(path.basename) + } + + func subtree(at path: WorkspacePath, node: MergeNode?) -> Snapshot.Entry { + let kind = node?.kind ?? .directory + switch kind { + case .file: + return .file( + Snapshot.File( + path: path, + data: node?.fileData ?? Data(), + permissions: node?.permissions ?? POSIXPermissions(0o644) + ) + ) + case .symlink: + return .symlink( + Snapshot.Symlink( + path: path, + target: node?.symlinkTarget ?? "", + permissions: node?.permissions ?? POSIXPermissions(0o777) + ) + ) + case .directory: + let children = (childNames[path] ?? []) + .sorted() + .map { name -> Snapshot.Entry in + let childPath = path.appending(name) + return subtree(at: childPath, node: nodes[childPath]) + } + return .directory( + Snapshot.Directory( + path: path, + permissions: node?.permissions ?? POSIXPermissions(0o755), + children: children + ) + ) + } + } + + return subtree(at: .root, node: MergeNode(kind: .directory, permissions: rootPermissions)) + } + func seedBranchCheckpoint( snapshot: Snapshot, label: String?, diff --git a/Tests/WorkspaceTests/ThreeWayMergeTests.swift b/Tests/WorkspaceTests/ThreeWayMergeTests.swift new file mode 100644 index 0000000..a0cb476 --- /dev/null +++ b/Tests/WorkspaceTests/ThreeWayMergeTests.swift @@ -0,0 +1,144 @@ +import Foundation +import Testing +@testable import Workspace + +@Suite("Three-way merge") +struct ThreeWayMergeTests { + @Test + func `non-overlapping changes on both sides merge cleanly`() async throws { + let workspace = Workspace(filesystem: InMemoryFilesystem()) + try await workspace.writeFile("/a.txt", content: "base a\n") + try await workspace.writeFile("/b.txt", content: "base b\n") + _ = try await workspace.createCheckpoint(label: "base") + + let branch = try await workspace.branch() + try await branch.writeFile("/b.txt", content: "branch b\n") + try await branch.writeFile("/new-branch.txt", content: "from branch\n") + + // Parent edits after branching, without a checkpoint — merge(_:) would refuse this. + try await workspace.writeFile("/a.txt", content: "parent a\n") + + let result = try await workspace.mergeThreeWay(branch, label: "merge branch") + + #expect(result.applied) + #expect(result.conflicts.isEmpty) + #expect(try await workspace.readFile("/a.txt") == "parent a\n") + #expect(try await workspace.readFile("/b.txt") == "branch b\n") + #expect(try await workspace.readFile("/new-branch.txt") == "from branch\n") + + let checkpoint = try #require(result.checkpoint) + #expect(checkpoint.mergedFromWorkspaceId == branch.workspaceId) + #expect((try await workspace.mutationRecords()).last?.kind == .mergeWorkspace) + } + + @Test + func `identical changes on both sides do not conflict`() async throws { + let workspace = Workspace(filesystem: InMemoryFilesystem()) + try await workspace.writeFile("/shared.txt", content: "base\n") + _ = try await workspace.createCheckpoint(label: "base") + + let branch = try await workspace.branch() + try await branch.writeFile("/shared.txt", content: "same change\n") + try await branch.writeFile("/dup.txt", content: "identical\n") + try await workspace.writeFile("/shared.txt", content: "same change\n") + try await workspace.writeFile("/dup.txt", content: "identical\n") + + let result = try await workspace.mergeThreeWay(branch) + + #expect(result.applied) + #expect(result.conflicts.isEmpty) + #expect(try await workspace.readFile("/shared.txt") == "same change\n") + } + + @Test + func `deletions on the branch propagate when ours is unchanged`() async throws { + let workspace = Workspace(filesystem: InMemoryFilesystem()) + try await workspace.createDirectory(at: "/doomed") + try await workspace.writeFile("/doomed/file.txt", content: "bye\n") + try await workspace.writeFile("/keep.txt", content: "keep\n") + _ = try await workspace.createCheckpoint(label: "base") + + let branch = try await workspace.branch() + try await branch.removeItem(at: "/doomed", recursive: true) + + let result = try await workspace.mergeThreeWay(branch) + + #expect(result.applied) + #expect(!(await workspace.exists("/doomed"))) + #expect(try await workspace.readFile("/keep.txt") == "keep\n") + } + + @Test + func `conflicting edits report bothModified with diffs and apply nothing`() async throws { + let workspace = Workspace(filesystem: InMemoryFilesystem()) + try await workspace.writeFile("/contested.txt", content: "base\n") + _ = try await workspace.createCheckpoint(label: "base") + let checkpointsBefore = try await workspace.listCheckpoints().count + + let branch = try await workspace.branch() + try await branch.writeFile("/contested.txt", content: "theirs\n") + try await workspace.writeFile("/contested.txt", content: "ours\n") + + let result = try await workspace.mergeThreeWay(branch) + + #expect(!result.applied) + #expect(result.checkpoint == nil) + #expect(result.conflicts.map(\.path) == ["/contested.txt"]) + #expect(result.conflicts.first?.kind == .bothModified) + #expect(result.conflicts.first?.oursDiff?.hunks.isEmpty == false) + #expect(result.conflicts.first?.theirsDiff?.hunks.isEmpty == false) + + // Nothing changed: ours content intact, no merge checkpoint or mutation appended. + #expect(try await workspace.readFile("/contested.txt") == "ours\n") + #expect(try await workspace.listCheckpoints().count == checkpointsBefore) + #expect((try await workspace.mutationRecords()).last?.kind != .mergeWorkspace) + } + + @Test + func `modify versus delete is reported as a conflict`() async throws { + let workspace = Workspace(filesystem: InMemoryFilesystem()) + try await workspace.writeFile("/contested.txt", content: "base\n") + _ = try await workspace.createCheckpoint(label: "base") + + let branch = try await workspace.branch() + try await branch.writeFile("/contested.txt", content: "modified\n") + try await workspace.removeItem(at: "/contested.txt") + + let result = try await workspace.mergeThreeWay(branch) + + #expect(!result.applied) + #expect(result.conflicts.map(\.kind) == [.modifiedAndDeleted]) + #expect(!(await workspace.exists("/contested.txt"))) + } + + @Test + func `divergent creations at the same path are reported as bothCreated`() async throws { + let workspace = Workspace(filesystem: InMemoryFilesystem()) + try await workspace.writeFile("/base.txt", content: "base\n") + _ = try await workspace.createCheckpoint(label: "base") + + let branch = try await workspace.branch() + try await branch.writeFile("/fresh.txt", content: "theirs\n") + try await workspace.writeFile("/fresh.txt", content: "ours\n") + + let result = try await workspace.mergeThreeWay(branch) + + #expect(!result.applied) + #expect(result.conflicts.map(\.kind) == [.bothCreated]) + #expect(try await workspace.readFile("/fresh.txt") == "ours\n") + } + + @Test + func `three-way merge requires a branch base checkpoint`() async throws { + let workspace = Workspace(filesystem: InMemoryFilesystem()) + try await workspace.writeFile("/a.txt", content: "no checkpoints yet") + let branch = try await workspace.branch() + + do { + _ = try await workspace.mergeThreeWay(branch) + Issue.record("expected unsupported error for a base-less branch") + } catch let error as WorkspaceError { + #expect(error.description.contains("three-way merge requires")) + } + } +}