Add Phase 4 duplicate verification proof fixture

Sources/StorageScopeBenchmark/main.swift

@@ -4,6 +4,7 @@ import StorageScopeCore
 struct BenchmarkArguments {
     var path: String?
     var useSyntheticFixture = false
+    var useDuplicateProofFixture = false
     var keepFixture = false
     var showFullPath = false
     /// User-requested file count for the synthetic fixture. 0 means "use the v0.5.0 curated
@@ -23,15 +24,21 @@ func usage() -> String {
       StorageScopeBenchmark [--show-full-path] [--repeat <n>] <folder>
       StorageScopeBenchmark --synthetic [--keep-fixture] [--show-full-path]
       StorageScopeBenchmark --synthetic --items <n> [--depth <d>] [--duplicates <0..1>] [--keep-fixture]
+      StorageScopeBenchmark --duplicate-proof [--repeat <n>] [--keep-fixture] [--show-full-path]
 
     Scaled fixtures (--items, --depth, --duplicates) build a synthetic tree of N files
     distributed across up to depth directory levels, with an optional fraction of
     duplicates (pairs sharing identical content). Useful for capturing v0.5.x perf
     baselines at 10k / 100k / 500k items.
 
+    The duplicate proof fixture models same-size media, DMGs, VM images, exact copies,
+    prefix collisions, and hard links. Its report includes the naive full-hash denominator
+    and measured byte reduction; use --repeat 2 to exercise the persisted warm cache.
+
     Examples:
       swift run StorageScopeBenchmark --synthetic --items 100000 --depth 8 --duplicates 0.2
       swift run StorageScopeBenchmark --synthetic --items 500000 --depth 12 --keep-fixture
+      swift run -c release StorageScopeBenchmark --duplicate-proof --repeat 2
       STORAGESCOPE_INCREMENTAL_RESCAN=1 swift run -c release StorageScopeBenchmark --repeat 3 <folder>
     """
 }
@@ -53,6 +60,8 @@ func parseArguments(_ rawArguments: [String]) throws -> BenchmarkArguments {
         switch value {
         case "--synthetic":
             arguments.useSyntheticFixture = true
+        case "--duplicate-proof":
+            arguments.useDuplicateProofFixture = true
         case "--keep-fixture":
             arguments.keepFixture = true
         case "--show-full-path":
@@ -100,6 +109,10 @@ func parseArguments(_ rawArguments: [String]) throws -> BenchmarkArguments {
         arguments.useSyntheticFixture = true
     }
 
+    if arguments.useSyntheticFixture && arguments.useDuplicateProofFixture {
+        throw BenchmarkError.invalidArgument("choose either --synthetic or --duplicate-proof")
+    }
+
     return arguments
 }
 
@@ -121,8 +134,18 @@ do {
     let arguments = try parseArguments(Array(CommandLine.arguments.dropFirst()))
     let rootURL: URL
     var cleanup: (() -> Void)?
+    var duplicateProofFixture: DuplicateVerificationProofFixture?
 
-    if arguments.useSyntheticFixture {
+    if arguments.useDuplicateProofFixture {
+        let fixture = try DuplicateVerificationProofFixture.create()
+        duplicateProofFixture = fixture
+        rootURL = fixture.rootURL
+        if arguments.keepFixture {
+            print("Duplicate proof fixture: \(rootURL.path)")
+        } else {
+            cleanup = { fixture.remove() }
+        }
+    } else if arguments.useSyntheticFixture {
         if arguments.items > 0 {
             let depth = arguments.depth > 0 ? arguments.depth : 5
             rootURL = try SyntheticBenchmarkFixture.create(
@@ -148,7 +171,21 @@ do {
         cleanup?()
     }
 
-    let runner = ScanBenchmarkRunner()
+    let runner: ScanBenchmarkRunner
+    var cacheCleanup: (() -> Void)?
+    if arguments.useDuplicateProofFixture {
+        let cacheDirectory = FileManager.default.temporaryDirectory.appendingPathComponent(
+            "StorageScopeDuplicateProofCache-\(UUID().uuidString)",
+            isDirectory: true
+        )
+        let cache = DuplicateHashCache(cacheURL: cacheDirectory.appendingPathComponent("hashes.json"))
+        runner = ScanBenchmarkRunner(scanner: FileSystemScanner(hashCache: cache), hashCache: cache)
+        cacheCleanup = { try? FileManager.default.removeItem(at: cacheDirectory) }
+    } else {
+        runner = ScanBenchmarkRunner()
+    }
+    defer { cacheCleanup?() }
+
     for run in 1...arguments.repeatCount {
         if arguments.repeatCount > 1 {
             print("Run \(run)/\(arguments.repeatCount)")
@@ -158,6 +195,14 @@ do {
             showFullPath: arguments.showFullPath
         )
         print(report.text)
+        if let fixture = duplicateProofFixture {
+            let naiveBytes = fixture.naiveFullHashBytes
+            let reduction = naiveBytes > 0
+                ? (1 - Double(report.duplicateVerificationBytesRead) / Double(naiveBytes)) * 100
+                : 0
+            print("Naive full-hash bytes: \(ByteCountFormatter.string(fromByteCount: naiveBytes, countStyle: .file))")
+            print(String(format: "Verification byte reduction: %.2f%%", reduction))
+        }
     }
 } catch {
     let message = (error as? LocalizedError)?.errorDescription ?? "\(error)"

Sources/StorageScopeCore/Models/StorageScan.swift

@@ -36,6 +36,7 @@ public struct StorageScan: Sendable {
     public let snapshotBuildCount: Int
     public let duplicateVerificationDuration: TimeInterval
     public let duplicateVerificationBytesRead: Int64
+    public let duplicateVerificationPeakOpenFiles: Int
     public let enumerateDuration: TimeInterval
     public let cleanupCandidates: [CleanupCandidate]
     public let isPartial: Bool
@@ -68,6 +69,7 @@ public struct StorageScan: Sendable {
         snapshotBuildCount: Int = 0,
         duplicateVerificationDuration: TimeInterval = 0,
         duplicateVerificationBytesRead: Int64 = 0,
+        duplicateVerificationPeakOpenFiles: Int = 0,
         enumerateDuration: TimeInterval = 0,
         cleanupCandidates: [CleanupCandidate],
         isPartial: Bool = false,
@@ -98,6 +100,7 @@ public struct StorageScan: Sendable {
         self.snapshotBuildCount = snapshotBuildCount
         self.duplicateVerificationDuration = duplicateVerificationDuration
         self.duplicateVerificationBytesRead = duplicateVerificationBytesRead
+        self.duplicateVerificationPeakOpenFiles = duplicateVerificationPeakOpenFiles
         self.enumerateDuration = enumerateDuration
         self.cleanupCandidates = cleanupCandidates
         self.isPartial = isPartial
@@ -144,6 +147,7 @@ public struct StorageScan: Sendable {
             snapshotBuildCount: 0,
             duplicateVerificationDuration: 0,
             duplicateVerificationBytesRead: 0,
+            duplicateVerificationPeakOpenFiles: 0,
             enumerateDuration: finishedAt.timeIntervalSince(startedAt),
             cleanupCandidates: cleanupCandidates,
             isPartial: false,

Sources/StorageScopeCore/Services/DuplicateVerificationProofFixture.swift

@@ -0,0 +1,143 @@
+import Foundation
+
+/// Reusable Phase 4 duplicate-verification corpus. The files are synthetic, but their
+/// extensions, size-group shape, early-divergence behavior, prefix collision, exact copies,
+/// and hard-link alias mirror the expensive media, disk-image, and VM cases the verifier
+/// encounters on real disks without committing large or copyrighted assets to the repo.
+public struct DuplicateVerificationProofFixture: Sendable {
+    public let rootURL: URL
+    public let naiveFullHashBytes: Int64
+    public let exactDuplicateURLs: [URL]
+    public let prefixCollisionURLs: [URL]
+    public let hardLinkURLs: [URL]
+
+    public var candidateFileCount: Int {
+        48 + exactDuplicateURLs.count + prefixCollisionURLs.count + hardLinkURLs.count
+    }
+
+    /// Creates a corpus with 48 same-size early-diverging media/DMG/VM candidates plus
+    /// exact-copy, prefix-collision, and hard-link groups. `largeCandidateBytes` defaults
+    /// to 4 MiB for release measurements; focused tests can use 1 MiB while preserving
+    /// the same >90% byte-reduction denominator.
+    public static func create(
+        in parentDirectory: URL = FileManager.default.temporaryDirectory,
+        fileManager: FileManager = .default,
+        largeCandidateBytes: Int = 4 * 1_024 * 1_024
+    ) throws -> DuplicateVerificationProofFixture {
+        let largeBytes = max(1_024 * 1_024, largeCandidateBytes)
+        let root = parentDirectory.appendingPathComponent(
+            "StorageScopeDuplicateProof-\(UUID().uuidString)",
+            isDirectory: true
+        )
+        let media = root.appendingPathComponent("Media", isDirectory: true)
+        let images = root.appendingPathComponent("Disk Images", isDirectory: true)
+        let virtualMachines = root.appendingPathComponent("Virtual Machines", isDirectory: true)
+        let collisions = root.appendingPathComponent("Prefix Collisions", isDirectory: true)
+        let links = root.appendingPathComponent("Hard Links", isDirectory: true)
+
+        do {
+            for directory in [media, images, virtualMachines, collisions, links] {
+                try fileManager.createDirectory(at: directory, withIntermediateDirectories: true)
+            }
+
+            let extensions = [
+                "mov", "mp4", "m4v", "dmg", "sparseimage", "vmdk", "qcow2", "utm"
+            ]
+            for index in 0..<48 {
+                let directory: URL
+                switch index % 3 {
+                case 0: directory = media
+                case 1: directory = images
+                default: directory = virtualMachines
+                }
+                let fileExtension = extensions[index % extensions.count]
+                let url = directory.appendingPathComponent(
+                    String(format: "candidate-%02d.%@", index, fileExtension)
+                )
+                try writePatternFile(
+                    at: url,
+                    bytes: largeBytes,
+                    prefixByte: UInt8(index + 1),
+                    bodyByte: UInt8((index * 5 + 17) % 251)
+                )
+            }
+
+            let exactDuplicateURLs = [
+                media.appendingPathComponent("export-copy-a.mov"),
+                virtualMachines.appendingPathComponent("export-copy-b.mov")
+            ]
+            for url in exactDuplicateURLs {
+                try writePatternFile(at: url, bytes: 128 * 1_024, prefixByte: 0xD1, bodyByte: 0xD2)
+            }
+
+            let prefixCollisionURLs = [
+                collisions.appendingPathComponent("shared-header-a.dmg"),
+                collisions.appendingPathComponent("shared-header-b.dmg")
+            ]
+            try writePatternFile(
+                at: prefixCollisionURLs[0],
+                bytes: 192 * 1_024,
+                prefixByte: 0xC1,
+                bodyByte: 0xC2
+            )
+            try writePatternFile(
+                at: prefixCollisionURLs[1],
+                bytes: 192 * 1_024,
+                prefixByte: 0xC1,
+                bodyByte: 0xC3
+            )
+
+            let hardLinkOriginal = links.appendingPathComponent("vm-base-original.img")
+            let hardLinkAlias = links.appendingPathComponent("vm-base-alias.img")
+            try writePatternFile(at: hardLinkOriginal, bytes: 96 * 1_024, prefixByte: 0xB1, bodyByte: 0xB2)
+            try fileManager.linkItem(at: hardLinkOriginal, to: hardLinkAlias)
+
+            let naiveBytes = Int64(48 * largeBytes) +
+                Int64(2 * 128 * 1_024) +
+                Int64(2 * 192 * 1_024) +
+                Int64(2 * 96 * 1_024)
+
+            return DuplicateVerificationProofFixture(
+                rootURL: root,
+                naiveFullHashBytes: naiveBytes,
+                exactDuplicateURLs: exactDuplicateURLs,
+                prefixCollisionURLs: prefixCollisionURLs,
+                hardLinkURLs: [hardLinkOriginal, hardLinkAlias]
+            )
+        } catch {
+            try? fileManager.removeItem(at: root)
+            throw error
+        }
+    }
+
+    public func remove(fileManager: FileManager = .default) {
+        try? fileManager.removeItem(at: rootURL)
+    }
+
+    private static func writePatternFile(
+        at url: URL,
+        bytes: Int,
+        prefixByte: UInt8,
+        bodyByte: UInt8
+    ) throws {
+        _ = FileManager.default.createFile(atPath: url.path, contents: nil)
+        let handle = try FileHandle(forWritingTo: url)
+        do {
+            let prefixCount = min(bytes, 64 * 1_024)
+            if prefixCount > 0 {
+                try handle.write(contentsOf: Data(repeating: prefixByte, count: prefixCount))
+            }
+            var remaining = bytes - prefixCount
+            let chunk = Data(repeating: bodyByte, count: min(max(remaining, 1), 1 * 1_024 * 1_024))
+            while remaining > 0 {
+                let count = min(remaining, chunk.count)
+                try handle.write(contentsOf: chunk.prefix(count))
+                remaining -= count
+            }
+            try handle.close()
+        } catch {
+            try? handle.close()
+            throw error
+        }
+    }
+}