From 03f3937d123edc2f736276679c4b6cee11a8ac06 Mon Sep 17 00:00:00 2001 From: RasputinKaiser <178525839+RasputinKaiser@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:52:00 -0400 Subject: [PATCH] Add Phase 4 duplicate verification proof fixture --- Sources/StorageScopeBenchmark/main.swift | 49 +++++- .../StorageScopeCore/Models/StorageScan.swift | 4 + .../DuplicateVerificationProofFixture.swift | 143 ++++++++++++++++++ .../Services/FileSystemScanner.swift | 79 ++++++++-- .../Services/ScanBenchmark.swift | 3 + .../DuplicateVerificationProofTests.swift | 117 ++++++++++++++ .../FileSystemScannerTests.swift | 1 + .../v0.7.1/phase4-duplicate-proof-fixture.md | 55 +++++++ 8 files changed, 435 insertions(+), 16 deletions(-) create mode 100644 Sources/StorageScopeCore/Services/DuplicateVerificationProofFixture.swift create mode 100644 Tests/StorageScopeCoreTests/DuplicateVerificationProofTests.swift create mode 100644 docs/perf-baselines/v0.7.1/phase4-duplicate-proof-fixture.md diff --git a/Sources/StorageScopeBenchmark/main.swift b/Sources/StorageScopeBenchmark/main.swift index fdb73ec..3f2b4c1 100644 --- a/Sources/StorageScopeBenchmark/main.swift +++ b/Sources/StorageScopeBenchmark/main.swift @@ -4,6 +4,7 @@ import StorageScopeCore struct BenchmarkArguments { var path: String? var useSyntheticFixture = false + var useDuplicateProofFixture = false var keepFixture = false var showFullPath = false /// User-requested file count for the synthetic fixture. 0 means "use the v0.5.0 curated @@ -23,15 +24,21 @@ func usage() -> String { StorageScopeBenchmark [--show-full-path] [--repeat ] StorageScopeBenchmark --synthetic [--keep-fixture] [--show-full-path] StorageScopeBenchmark --synthetic --items [--depth ] [--duplicates <0..1>] [--keep-fixture] + StorageScopeBenchmark --duplicate-proof [--repeat ] [--keep-fixture] [--show-full-path] Scaled fixtures (--items, --depth, --duplicates) build a synthetic tree of N files distributed across up to depth directory levels, with an optional fraction of duplicates (pairs sharing identical content). Useful for capturing v0.5.x perf baselines at 10k / 100k / 500k items. + The duplicate proof fixture models same-size media, DMGs, VM images, exact copies, + prefix collisions, and hard links. Its report includes the naive full-hash denominator + and measured byte reduction; use --repeat 2 to exercise the persisted warm cache. + Examples: swift run StorageScopeBenchmark --synthetic --items 100000 --depth 8 --duplicates 0.2 swift run StorageScopeBenchmark --synthetic --items 500000 --depth 12 --keep-fixture + swift run -c release StorageScopeBenchmark --duplicate-proof --repeat 2 STORAGESCOPE_INCREMENTAL_RESCAN=1 swift run -c release StorageScopeBenchmark --repeat 3 """ } @@ -53,6 +60,8 @@ func parseArguments(_ rawArguments: [String]) throws -> BenchmarkArguments { switch value { case "--synthetic": arguments.useSyntheticFixture = true + case "--duplicate-proof": + arguments.useDuplicateProofFixture = true case "--keep-fixture": arguments.keepFixture = true case "--show-full-path": @@ -100,6 +109,10 @@ func parseArguments(_ rawArguments: [String]) throws -> BenchmarkArguments { arguments.useSyntheticFixture = true } + if arguments.useSyntheticFixture && arguments.useDuplicateProofFixture { + throw BenchmarkError.invalidArgument("choose either --synthetic or --duplicate-proof") + } + return arguments } @@ -121,8 +134,18 @@ do { let arguments = try parseArguments(Array(CommandLine.arguments.dropFirst())) let rootURL: URL var cleanup: (() -> Void)? + var duplicateProofFixture: DuplicateVerificationProofFixture? - if arguments.useSyntheticFixture { + if arguments.useDuplicateProofFixture { + let fixture = try DuplicateVerificationProofFixture.create() + duplicateProofFixture = fixture + rootURL = fixture.rootURL + if arguments.keepFixture { + print("Duplicate proof fixture: \(rootURL.path)") + } else { + cleanup = { fixture.remove() } + } + } else if arguments.useSyntheticFixture { if arguments.items > 0 { let depth = arguments.depth > 0 ? arguments.depth : 5 rootURL = try SyntheticBenchmarkFixture.create( @@ -148,7 +171,21 @@ do { cleanup?() } - let runner = ScanBenchmarkRunner() + let runner: ScanBenchmarkRunner + var cacheCleanup: (() -> Void)? + if arguments.useDuplicateProofFixture { + let cacheDirectory = FileManager.default.temporaryDirectory.appendingPathComponent( + "StorageScopeDuplicateProofCache-\(UUID().uuidString)", + isDirectory: true + ) + let cache = DuplicateHashCache(cacheURL: cacheDirectory.appendingPathComponent("hashes.json")) + runner = ScanBenchmarkRunner(scanner: FileSystemScanner(hashCache: cache), hashCache: cache) + cacheCleanup = { try? FileManager.default.removeItem(at: cacheDirectory) } + } else { + runner = ScanBenchmarkRunner() + } + defer { cacheCleanup?() } + for run in 1...arguments.repeatCount { if arguments.repeatCount > 1 { print("Run \(run)/\(arguments.repeatCount)") @@ -158,6 +195,14 @@ do { showFullPath: arguments.showFullPath ) print(report.text) + if let fixture = duplicateProofFixture { + let naiveBytes = fixture.naiveFullHashBytes + let reduction = naiveBytes > 0 + ? (1 - Double(report.duplicateVerificationBytesRead) / Double(naiveBytes)) * 100 + : 0 + print("Naive full-hash bytes: \(ByteCountFormatter.string(fromByteCount: naiveBytes, countStyle: .file))") + print(String(format: "Verification byte reduction: %.2f%%", reduction)) + } } } catch { let message = (error as? LocalizedError)?.errorDescription ?? "\(error)" diff --git a/Sources/StorageScopeCore/Models/StorageScan.swift b/Sources/StorageScopeCore/Models/StorageScan.swift index 7fcbf85..1c77cbe 100644 --- a/Sources/StorageScopeCore/Models/StorageScan.swift +++ b/Sources/StorageScopeCore/Models/StorageScan.swift @@ -36,6 +36,7 @@ public struct StorageScan: Sendable { public let snapshotBuildCount: Int public let duplicateVerificationDuration: TimeInterval public let duplicateVerificationBytesRead: Int64 + public let duplicateVerificationPeakOpenFiles: Int public let enumerateDuration: TimeInterval public let cleanupCandidates: [CleanupCandidate] public let isPartial: Bool @@ -68,6 +69,7 @@ public struct StorageScan: Sendable { snapshotBuildCount: Int = 0, duplicateVerificationDuration: TimeInterval = 0, duplicateVerificationBytesRead: Int64 = 0, + duplicateVerificationPeakOpenFiles: Int = 0, enumerateDuration: TimeInterval = 0, cleanupCandidates: [CleanupCandidate], isPartial: Bool = false, @@ -98,6 +100,7 @@ public struct StorageScan: Sendable { self.snapshotBuildCount = snapshotBuildCount self.duplicateVerificationDuration = duplicateVerificationDuration self.duplicateVerificationBytesRead = duplicateVerificationBytesRead + self.duplicateVerificationPeakOpenFiles = duplicateVerificationPeakOpenFiles self.enumerateDuration = enumerateDuration self.cleanupCandidates = cleanupCandidates self.isPartial = isPartial @@ -144,6 +147,7 @@ public struct StorageScan: Sendable { snapshotBuildCount: 0, duplicateVerificationDuration: 0, duplicateVerificationBytesRead: 0, + duplicateVerificationPeakOpenFiles: 0, enumerateDuration: finishedAt.timeIntervalSince(startedAt), cleanupCandidates: cleanupCandidates, isPartial: false, diff --git a/Sources/StorageScopeCore/Services/DuplicateVerificationProofFixture.swift b/Sources/StorageScopeCore/Services/DuplicateVerificationProofFixture.swift new file mode 100644 index 0000000..9f30ec9 --- /dev/null +++ b/Sources/StorageScopeCore/Services/DuplicateVerificationProofFixture.swift @@ -0,0 +1,143 @@ +import Foundation + +/// Reusable Phase 4 duplicate-verification corpus. The files are synthetic, but their +/// extensions, size-group shape, early-divergence behavior, prefix collision, exact copies, +/// and hard-link alias mirror the expensive media, disk-image, and VM cases the verifier +/// encounters on real disks without committing large or copyrighted assets to the repo. +public struct DuplicateVerificationProofFixture: Sendable { + public let rootURL: URL + public let naiveFullHashBytes: Int64 + public let exactDuplicateURLs: [URL] + public let prefixCollisionURLs: [URL] + public let hardLinkURLs: [URL] + + public var candidateFileCount: Int { + 48 + exactDuplicateURLs.count + prefixCollisionURLs.count + hardLinkURLs.count + } + + /// Creates a corpus with 48 same-size early-diverging media/DMG/VM candidates plus + /// exact-copy, prefix-collision, and hard-link groups. `largeCandidateBytes` defaults + /// to 4 MiB for release measurements; focused tests can use 1 MiB while preserving + /// the same >90% byte-reduction denominator. + public static func create( + in parentDirectory: URL = FileManager.default.temporaryDirectory, + fileManager: FileManager = .default, + largeCandidateBytes: Int = 4 * 1_024 * 1_024 + ) throws -> DuplicateVerificationProofFixture { + let largeBytes = max(1_024 * 1_024, largeCandidateBytes) + let root = parentDirectory.appendingPathComponent( + "StorageScopeDuplicateProof-\(UUID().uuidString)", + isDirectory: true + ) + let media = root.appendingPathComponent("Media", isDirectory: true) + let images = root.appendingPathComponent("Disk Images", isDirectory: true) + let virtualMachines = root.appendingPathComponent("Virtual Machines", isDirectory: true) + let collisions = root.appendingPathComponent("Prefix Collisions", isDirectory: true) + let links = root.appendingPathComponent("Hard Links", isDirectory: true) + + do { + for directory in [media, images, virtualMachines, collisions, links] { + try fileManager.createDirectory(at: directory, withIntermediateDirectories: true) + } + + let extensions = [ + "mov", "mp4", "m4v", "dmg", "sparseimage", "vmdk", "qcow2", "utm" + ] + for index in 0..<48 { + let directory: URL + switch index % 3 { + case 0: directory = media + case 1: directory = images + default: directory = virtualMachines + } + let fileExtension = extensions[index % extensions.count] + let url = directory.appendingPathComponent( + String(format: "candidate-%02d.%@", index, fileExtension) + ) + try writePatternFile( + at: url, + bytes: largeBytes, + prefixByte: UInt8(index + 1), + bodyByte: UInt8((index * 5 + 17) % 251) + ) + } + + let exactDuplicateURLs = [ + media.appendingPathComponent("export-copy-a.mov"), + virtualMachines.appendingPathComponent("export-copy-b.mov") + ] + for url in exactDuplicateURLs { + try writePatternFile(at: url, bytes: 128 * 1_024, prefixByte: 0xD1, bodyByte: 0xD2) + } + + let prefixCollisionURLs = [ + collisions.appendingPathComponent("shared-header-a.dmg"), + collisions.appendingPathComponent("shared-header-b.dmg") + ] + try writePatternFile( + at: prefixCollisionURLs[0], + bytes: 192 * 1_024, + prefixByte: 0xC1, + bodyByte: 0xC2 + ) + try writePatternFile( + at: prefixCollisionURLs[1], + bytes: 192 * 1_024, + prefixByte: 0xC1, + bodyByte: 0xC3 + ) + + let hardLinkOriginal = links.appendingPathComponent("vm-base-original.img") + let hardLinkAlias = links.appendingPathComponent("vm-base-alias.img") + try writePatternFile(at: hardLinkOriginal, bytes: 96 * 1_024, prefixByte: 0xB1, bodyByte: 0xB2) + try fileManager.linkItem(at: hardLinkOriginal, to: hardLinkAlias) + + let naiveBytes = Int64(48 * largeBytes) + + Int64(2 * 128 * 1_024) + + Int64(2 * 192 * 1_024) + + Int64(2 * 96 * 1_024) + + return DuplicateVerificationProofFixture( + rootURL: root, + naiveFullHashBytes: naiveBytes, + exactDuplicateURLs: exactDuplicateURLs, + prefixCollisionURLs: prefixCollisionURLs, + hardLinkURLs: [hardLinkOriginal, hardLinkAlias] + ) + } catch { + try? fileManager.removeItem(at: root) + throw error + } + } + + public func remove(fileManager: FileManager = .default) { + try? fileManager.removeItem(at: rootURL) + } + + private static func writePatternFile( + at url: URL, + bytes: Int, + prefixByte: UInt8, + bodyByte: UInt8 + ) throws { + _ = FileManager.default.createFile(atPath: url.path, contents: nil) + let handle = try FileHandle(forWritingTo: url) + do { + let prefixCount = min(bytes, 64 * 1_024) + if prefixCount > 0 { + try handle.write(contentsOf: Data(repeating: prefixByte, count: prefixCount)) + } + var remaining = bytes - prefixCount + let chunk = Data(repeating: bodyByte, count: min(max(remaining, 1), 1 * 1_024 * 1_024)) + while remaining > 0 { + let count = min(remaining, chunk.count) + try handle.write(contentsOf: chunk.prefix(count)) + remaining -= count + } + try handle.close() + } catch { + try? handle.close() + throw error + } + } +} diff --git a/Sources/StorageScopeCore/Services/FileSystemScanner.swift b/Sources/StorageScopeCore/Services/FileSystemScanner.swift index cc842ac..db0e68b 100644 --- a/Sources/StorageScopeCore/Services/FileSystemScanner.swift +++ b/Sources/StorageScopeCore/Services/FileSystemScanner.swift @@ -1,6 +1,7 @@ import CryptoKit import Foundation import os +import os.lock public enum FileSystemScannerError: LocalizedError { case cancelled @@ -249,6 +250,7 @@ public final class FileSystemScanner { ) let duplicateVerificationDuration = Date().timeIntervalSince(duplicateVerificationStartedAt) let duplicateVerificationBytesRead = accumulator.duplicateVerificationBytesRead + let duplicateVerificationPeakOpenFiles = accumulator.duplicateVerificationPeakOpenFiles let finishedAt = Date() os_signpost(.end, log: Self.log, name: "scan", signpostID: Self.signpostID, @@ -278,6 +280,7 @@ public final class FileSystemScanner { snapshotBuildCount: accumulator.snapshotBuildCount, duplicateVerificationDuration: duplicateVerificationDuration, duplicateVerificationBytesRead: duplicateVerificationBytesRead, + duplicateVerificationPeakOpenFiles: duplicateVerificationPeakOpenFiles, enumerateDuration: enumerateDuration, cleanupCandidates: accumulator.cleanupCandidates( rootID: rootItem.id, @@ -409,12 +412,12 @@ public final class FileSystemScanner { os_signpost(.end, log: Self.log, name: "verify_on_demand", signpostID: verifySignpostID) } - let ioSemaphore = DispatchSemaphore(value: Self.hashConcurrency) + let ioLimiter = DuplicateHashReadLimiter(capacity: Self.hashConcurrency) let cacheLock = NSLock() return try verifiedDuplicateGroups( in: group, - ioSemaphore: ioSemaphore, + ioLimiter: ioLimiter, cacheLock: cacheLock, recordBytesRead: nil, cancellation: cancellation @@ -1176,7 +1179,10 @@ public final class FileSystemScanner { let verifiedGroupsLock = NSLock() let cacheLock = NSLock() - let ioSemaphore = DispatchSemaphore(value: Self.hashConcurrency) + let ioLimiter = DuplicateHashReadLimiter(capacity: Self.hashConcurrency) + defer { + accumulator.recordDuplicateVerificationPeakOpenFiles(ioLimiter.peakOpenFiles) + } // Cancellation is cooperative: concurrentPerform can't throw, so iterations record // observed cancellation in a flag that we re-check after the call returns. This // mirrors the pattern used by the parallel-directory-enumeration path in scanItem. @@ -1211,7 +1217,7 @@ public final class FileSystemScanner { let verifiedForSize = try verifiedDuplicateGroups( in: sizeGroup, - ioSemaphore: ioSemaphore, + ioLimiter: ioLimiter, cacheLock: cacheLock, recordBytesRead: accumulator.recordDuplicateVerificationBytes, cancellation: cancellation @@ -1247,7 +1253,7 @@ public final class FileSystemScanner { private func verifiedDuplicateGroups( in sizeGroup: DuplicateSizeGroup, - ioSemaphore: DispatchSemaphore, + ioLimiter: DuplicateHashReadLimiter, cacheLock: NSLock, recordBytesRead: ((Int) -> Void)?, cancellation: ScanCancellation? @@ -1282,7 +1288,7 @@ public final class FileSystemScanner { if let prefixed = try prefixHashedFormItem( item, - ioSemaphore: ioSemaphore, + ioLimiter: ioLimiter, recordBytesRead: recordBytesRead, cancellation: cancellation ) { @@ -1313,7 +1319,7 @@ public final class FileSystemScanner { if let hashed = try hashedFormItem( prefixed.item, - ioSemaphore: ioSemaphore, + ioLimiter: ioLimiter, cacheLock: cacheLock, recordBytesRead: recordBytesRead, cancellation: cancellation @@ -1458,12 +1464,12 @@ public final class FileSystemScanner { private func prefixHashedFormItem( _ item: StorageItem, - ioSemaphore: DispatchSemaphore, + ioLimiter: DuplicateHashReadLimiter, recordBytesRead: ((Int) -> Void)?, cancellation: ScanCancellation? ) throws -> PrefixHashedStorageItem? { - ioSemaphore.wait() - defer { ioSemaphore.signal() } + ioLimiter.wait() + defer { ioLimiter.signal() } do { let result = try prefixChecksum( @@ -1487,13 +1493,13 @@ public final class FileSystemScanner { } /// Hashes one item, consulting the persisted `hashCache` fast-path before falling back - /// to a full `sha256Checksum` read. I/O is throttled through `ioSemaphore`; cache writes + /// to a full `sha256Checksum` read. I/O is throttled through `ioLimiter`; cache writes /// are serialised through `cacheLock`. Returns `nil` for per-file read failures (logged /// via os_signpost so Instruments shows what was skipped), and re-throws /// `FileSystemScannerError.cancelled` so the caller can abort the whole batch. private func hashedFormItem( _ item: StorageItem, - ioSemaphore: DispatchSemaphore, + ioLimiter: DuplicateHashReadLimiter, cacheLock: NSLock, recordBytesRead: ((Int) -> Void)?, cancellation: ScanCancellation? @@ -1504,8 +1510,8 @@ public final class FileSystemScanner { return HashedStorageItem(checksum: cached, item: item) } - ioSemaphore.wait() - defer { ioSemaphore.signal() } + ioLimiter.wait() + defer { ioLimiter.signal() } do { let checksum = try sha256Checksum( @@ -1529,6 +1535,43 @@ public final class FileSystemScanner { } } +/// Couples the duplicate verifier's file-handle budget with a measured peak so release +/// benchmarks can prove descriptor use stayed bounded instead of inferring it from a +/// semaphore constant. A permit is acquired immediately before opening a `FileHandle` +/// and released after the handle's deferred close path has run. +private final class DuplicateHashReadLimiter: @unchecked Sendable { + private struct State { + var activeOpenFiles = 0 + var peakOpenFiles = 0 + } + + private let semaphore: DispatchSemaphore + private let state = OSAllocatedUnfairLock(initialState: State()) + + init(capacity: Int) { + semaphore = DispatchSemaphore(value: max(1, capacity)) + } + + func wait() { + semaphore.wait() + state.withLock { state in + state.activeOpenFiles += 1 + state.peakOpenFiles = max(state.peakOpenFiles, state.activeOpenFiles) + } + } + + func signal() { + state.withLock { state in + state.activeOpenFiles = max(0, state.activeOpenFiles - 1) + } + semaphore.signal() + } + + var peakOpenFiles: Int { + state.withLock { $0.peakOpenFiles } + } +} + private extension StorageItem.Kind { var isDirectoryKind: Bool { self == .folder || self == .package @@ -2524,6 +2567,7 @@ private final class ScanAccumulator { var inaccessibleItemCount = 0 var totalBytes: Int64 = 0 private(set) var duplicateVerificationBytesRead: Int64 = 0 + private(set) var duplicateVerificationPeakOpenFiles = 0 private(set) var snapshotBuildCount = 0 /// Guards every mutable field above. Held briefly during directory enumeration's @@ -2785,6 +2829,13 @@ private final class ScanAccumulator { lock.unlock() } + func recordDuplicateVerificationPeakOpenFiles(_ count: Int) { + guard count > 0 else { return } + lock.lock() + duplicateVerificationPeakOpenFiles = max(duplicateVerificationPeakOpenFiles, count) + lock.unlock() + } + func recordPhase(path: String, phase: ScanPhase = .enumerating) { lock.lock() defer { lock.unlock() } diff --git a/Sources/StorageScopeCore/Services/ScanBenchmark.swift b/Sources/StorageScopeCore/Services/ScanBenchmark.swift index 5a8e1bd..6c6492e 100644 --- a/Sources/StorageScopeCore/Services/ScanBenchmark.swift +++ b/Sources/StorageScopeCore/Services/ScanBenchmark.swift @@ -18,6 +18,7 @@ public struct ScanBenchmarkReport: Hashable, Sendable { public let verifiedDuplicateGroupCount: Int public let duplicateVerificationDuration: TimeInterval public let duplicateVerificationBytesRead: Int64 + public let duplicateVerificationPeakOpenFiles: Int public let enumerateDuration: TimeInterval public let verifyDuration: TimeInterval public let persistDuration: TimeInterval @@ -56,6 +57,7 @@ public struct ScanBenchmarkReport: Hashable, Sendable { self.verifiedDuplicateGroupCount = scan.verifiedDuplicateGroups.count self.duplicateVerificationDuration = scan.duplicateVerificationDuration self.duplicateVerificationBytesRead = scan.duplicateVerificationBytesRead + self.duplicateVerificationPeakOpenFiles = scan.duplicateVerificationPeakOpenFiles self.enumerateDuration = scan.enumerateDuration self.verifyDuration = scan.duplicateVerificationDuration self.persistDuration = persistDuration @@ -84,6 +86,7 @@ public struct ScanBenchmarkReport: Hashable, Sendable { "Verified duplicate groups: \(verifiedDuplicateGroupCount.formatted())", "Duplicate verification: \(Self.seconds(duplicateVerificationDuration))", "Duplicate verification bytes read: \(Self.bytes(duplicateVerificationBytesRead))", + "Duplicate verification peak open files: \(duplicateVerificationPeakOpenFiles.formatted())", "Enumerate duration: \(Self.seconds(enumerateDuration))", "Verify duration: \(Self.seconds(verifyDuration))", "Persist duration: \(Self.seconds(persistDuration))", diff --git a/Tests/StorageScopeCoreTests/DuplicateVerificationProofTests.swift b/Tests/StorageScopeCoreTests/DuplicateVerificationProofTests.swift new file mode 100644 index 0000000..bf091b4 --- /dev/null +++ b/Tests/StorageScopeCoreTests/DuplicateVerificationProofTests.swift @@ -0,0 +1,117 @@ +import Foundation +import Testing +@testable import StorageScopeCore + +@Suite("Phase 4 duplicate verification proof", .serialized) +struct DuplicateVerificationProofTests { + @Test("realistic corpus measures byte, cache, hard-link, mutation, cancellation, and descriptor gates") + func realisticCorpusEstablishesAcceptanceSurface() throws { + let fixture = try DuplicateVerificationProofFixture.create(largeCandidateBytes: 1 * 1_024 * 1_024) + defer { fixture.remove() } + + #expect(fixture.candidateFileCount == 54) + #expect(fixture.naiveFullHashBytes > 48 * 1_000_000) + let extensions = try Set( + FileManager.default.subpathsOfDirectory(atPath: fixture.rootURL.path) + .map { URL(fileURLWithPath: $0).pathExtension.lowercased() } + ) + #expect(extensions.isSuperset(of: ["mov", "mp4", "dmg", "vmdk", "qcow2", "img"])) + + let originalAttributes = try FileManager.default.attributesOfItem(atPath: fixture.hardLinkURLs[0].path) + let aliasAttributes = try FileManager.default.attributesOfItem(atPath: fixture.hardLinkURLs[1].path) + #expect(originalAttributes[.systemFileNumber] as? NSNumber == aliasAttributes[.systemFileNumber] as? NSNumber) + #expect((originalAttributes[.referenceCount] as? NSNumber)?.intValue ?? 0 >= 2) + + let cacheDirectory = FileManager.default.temporaryDirectory.appendingPathComponent( + "StorageScopeDuplicateProofCacheTests-\(UUID().uuidString)", + isDirectory: true + ) + defer { try? FileManager.default.removeItem(at: cacheDirectory) } + let cacheURL = cacheDirectory.appendingPathComponent("hashes.json") + let coldCache = DuplicateHashCache(cacheURL: cacheURL) + let options = ScanOptions.benchmarkDefaults() + let coldScan = try FileSystemScanner(hashCache: coldCache).scan( + root: fixture.rootURL, + options: options + ) + + let maximumProofBytes = fixture.naiveFullHashBytes / 10 + #expect(coldScan.duplicateVerificationBytesRead <= maximumProofBytes) + #expect(coldScan.duplicateVerificationPeakOpenFiles > 0) + #expect(coldScan.duplicateVerificationPeakOpenFiles <= 6) + + let exactDuplicateNames = Set(fixture.exactDuplicateURLs.map(\.lastPathComponent)) + let verifiedNameSets = coldScan.verifiedDuplicateGroups.map { Set($0.items.map(\.name)) } + #expect(verifiedNameSets.contains(exactDuplicateNames)) + + let hardLinkPaths = Set(fixture.hardLinkURLs.map { $0.standardizedFileURL.path }) + let candidatePaths = Set(coldScan.duplicateSizeGroups.flatMap(\.items).map { $0.url.standardizedFileURL.path }) + let verifiedPaths = Set(coldScan.verifiedDuplicateGroups.flatMap(\.items).map { $0.url.standardizedFileURL.path }) + let hardLinksExcludedFromCandidates = candidatePaths.isDisjoint(with: hardLinkPaths) + let hardLinksExcludedFromVerifiedGroups = verifiedPaths.isDisjoint(with: hardLinkPaths) + withKnownIssue("Phase 4 must exclude hard-link aliases before duplicate verification") { + #expect(hardLinksExcludedFromCandidates) + #expect(hardLinksExcludedFromVerifiedGroups) + } + + coldCache.persist() + let warmCache = DuplicateHashCache(cacheURL: cacheURL) + let warmScan = try FileSystemScanner(hashCache: warmCache).scan( + root: fixture.rootURL, + options: options + ) + #expect(warmScan.duplicateVerificationBytesRead < coldScan.duplicateVerificationBytesRead) + withKnownIssue("Phase 4 must persist prefix digests so a cold-process warm cache performs zero reads") { + #expect(warmScan.duplicateVerificationBytesRead == 0) + } + + let exactGroup = try #require( + coldScan.duplicateSizeGroups.first { group in + Set(group.items.map(\.name)) == exactDuplicateNames + } + ) + let changedURL = fixture.exactDuplicateURLs[1] + try FileManager.default.setAttributes( + [.modificationDate: Date(timeIntervalSinceNow: 120)], + ofItemAtPath: changedURL.path + ) + let changedResult = try FileSystemScanner().verifySizeGroup(exactGroup) + let changedFileWasRejected = changedResult.isEmpty + withKnownIssue("Phase 4 must reject a file whose metadata changed after enumeration and before hashing") { + #expect(changedFileWasRejected) + } + + let largeGroup = try #require( + coldScan.duplicateSizeGroups.first { $0.byteSize == 1 * 1_024 * 1_024 } + ) + let cancellation = ScanCancellation() + cancellation.cancel() + #expect(throws: FileSystemScannerError.self) { + _ = try FileSystemScanner().verifySizeGroup(largeGroup, cancellation: cancellation) + } + } + + @Test("benchmark CLI exposes the reusable duplicate proof mode") + func benchmarkCLIExposesDuplicateProofMode() throws { + let repoRoot = try repositoryRoot() + let source = try String( + contentsOf: repoRoot.appendingPathComponent("Sources/StorageScopeBenchmark/main.swift"), + encoding: .utf8 + ) + + #expect(source.contains("--duplicate-proof")) + #expect(source.contains("Naive full-hash bytes:")) + #expect(source.contains("Verification byte reduction:")) + } + + private func repositoryRoot() throws -> URL { + var url = URL(fileURLWithPath: #filePath) + while url.pathComponents.count > 1 { + url.deleteLastPathComponent() + if FileManager.default.fileExists(atPath: url.appendingPathComponent("Package.swift").path) { + return url + } + } + throw CocoaError(.fileNoSuchFile) + } +} diff --git a/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift b/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift index fa4c162..16ef38a 100644 --- a/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift +++ b/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift @@ -1040,6 +1040,7 @@ struct FileSystemScannerTests { #expect(text.contains("Duplicate candidates:")) #expect(text.contains("Duplicate evictions:")) #expect(text.contains("Duplicate verification:")) + #expect(text.contains("Duplicate verification peak open files:")) #expect(text.contains("Snapshots built:")) #expect(text.contains("Results are local only.")) #expect(!text.contains(temporaryRoot.deletingLastPathComponent().path)) diff --git a/docs/perf-baselines/v0.7.1/phase4-duplicate-proof-fixture.md b/docs/perf-baselines/v0.7.1/phase4-duplicate-proof-fixture.md new file mode 100644 index 0000000..dcb33bb --- /dev/null +++ b/docs/perf-baselines/v0.7.1/phase4-duplicate-proof-fixture.md @@ -0,0 +1,55 @@ +# Phase 4 duplicate-verification proof fixture + +Date: 2026-07-13 +Commit base: `4fa1213` (post-Phase-3 `main`) +Machine: Apple silicon Mac +Build: Swift release +Regime: freshly generated local temporary fixture; run 1 cold hash cache, run 2 +persisted warm hash cache + +Command: + +```sh +./script/benchmark_scan.sh --duplicate-proof --repeat 2 +``` + +The reusable corpus contains 48 same-size early-diverging media, DMG, and VM +files, one exact-copy pair, one same-prefix/different-tail collision pair, and one +hard-link pair. It is 202.2 MB across 54 duplicate candidates. The naive baseline +is the sum of every candidate byte that a full-hash-only verifier would read. + +| Measurement | Cold run | Warm persisted-cache run | +|---|---:|---:| +| Duration | 0.02 s | 0.01 s | +| Naive full-hash bytes | 202.2 MB | 202.2 MB | +| Verification bytes read | 4.4 MB | 3.1 MB | +| Byte reduction | 97.83% | 98.44% | +| Peak open verification files | 3 | 1 | +| Verified groups | 2 | 2 | +| Peak RSS | 18.5 MB | 22.3 MB | + +The cold byte-reduction target and six-reader descriptor ceiling pass on this +fixture. The warm run is near-instant, but it still reads 3,145,728 bytes because +prefix digests are not persisted yet. + +The focused acceptance suite is: + +```sh +swift test --scratch-path /tmp/storagescope-phase4-proof-focused \ + --disable-index-store --jobs 1 --filter DuplicateVerificationProofTests +``` + +It passes with four recorded known-issue assertions covering three open Phase 4 +gaps: + +- hard-link aliases remain in candidate and verified groups; +- a cold-process warm cache still reads the 48 large-file prefixes; +- a file whose modification time changes after enumeration and before hashing is + still accepted. + +Cancellation propagates as `FileSystemScannerError.cancelled` without returning a +partial group. The mutation proof currently covers the enumeration-to-hash +boundary; deterministic mutation during an active read remains part of the Phase 4 +implementation tranche. Corrupt-cache quarantine, versioned/batched persistence, +raw digests, reusable buffers, and volume-aware read sizing are not claimed by this +fixture-only tranche.