Fix DiffID computation to use uncompressed layer digest

Sources/ContainerizationOCI/Content/ContentWriter.swift

@@ -14,10 +14,12 @@
 // limitations under the License.
 //===----------------------------------------------------------------------===//
 
+import Compression
 import ContainerizationError
 import Crypto
 import Foundation
 import NIOCore
+import zlib
 
 /// Provides a context to write data into a directory.
 public class ContentWriter {
@@ -134,4 +136,168 @@ public class ContentWriter {
         let data = try self.encoder.encode(content)
         return try self.write(data)
     }
+
+    /// Computes the SHA256 digest of the uncompressed content of a gzip file.
+    ///
+    /// Per the OCI Image Specification, a DiffID is the SHA256 digest of the
+    /// uncompressed layer content. This method streams the compressed file in
+    /// chunks, decompresses through Apple's Compression framework, and feeds
+    /// each decompressed chunk into an incremental SHA256 hasher. Neither the
+    /// full compressed nor the full decompressed data is held in memory.
+    ///
+    /// - Parameter url: The URL of the gzip-compressed file.
+    /// - Returns: The SHA256 digest of the uncompressed content.
+    public static func diffID(of url: URL) throws -> SHA256.Digest {
+        let fileHandle = try FileHandle(forReadingFrom: url)
+        defer { fileHandle.closeFile() }
+
+        let headerReadSize = 512
+        guard let headerData = Self.readExactly(fileHandle: fileHandle, count: headerReadSize),
+            !headerData.isEmpty
+        else {
+            throw ContainerizationError(.internalError, message: "invalid gzip file")
+        }
+        let headerSize = try Self.gzipHeaderSize(headerData)
+
+        fileHandle.seekToEndOfFile()
+        let fileSize = fileHandle.offsetInFile
+        guard fileSize >= 8 else {
+            throw ContainerizationError(.internalError, message: "gzip trailer mismatch")
+        }
+        fileHandle.seek(toFileOffset: fileSize - 8)
+        guard let trailerData = Self.readExactly(fileHandle: fileHandle, count: 8),
+            trailerData.count == 8
+        else {
+            throw ContainerizationError(.internalError, message: "gzip trailer mismatch")
+        }
+        let expectedCRC =
+            UInt32(trailerData[trailerData.startIndex])
+            | (UInt32(trailerData[trailerData.startIndex + 1]) << 8)
+            | (UInt32(trailerData[trailerData.startIndex + 2]) << 16)
+            | (UInt32(trailerData[trailerData.startIndex + 3]) << 24)
+        let expectedSize =
+            UInt32(trailerData[trailerData.startIndex + 4])
+            | (UInt32(trailerData[trailerData.startIndex + 5]) << 8)
+            | (UInt32(trailerData[trailerData.startIndex + 6]) << 16)
+            | (UInt32(trailerData[trailerData.startIndex + 7]) << 24)
+
+        fileHandle.seek(toFileOffset: UInt64(headerSize))
+        var compressedBytesRemaining = Int(fileSize) - headerSize - 8
+        guard compressedBytesRemaining >= 0 else {
+            throw ContainerizationError(.internalError, message: "invalid gzip file")
+        }
+
+        let chunkSize = 65_536
+        let sourceBuffer = UnsafeMutablePointer<UInt8>.allocate(capacity: chunkSize)
+        let destinationBuffer = UnsafeMutablePointer<UInt8>.allocate(capacity: chunkSize)
+        defer {
+            sourceBuffer.deallocate()
+            destinationBuffer.deallocate()
+        }
+
+        let stream = UnsafeMutablePointer<compression_stream>.allocate(capacity: 1)
+        defer { stream.deallocate() }
+
+        var status = compression_stream_init(stream, COMPRESSION_STREAM_DECODE, COMPRESSION_ZLIB)
+        guard status != COMPRESSION_STATUS_ERROR else {
+            throw ContainerizationError(.internalError, message: "gzip decompression failed")
+        }
+        defer { compression_stream_destroy(stream) }
+
+        stream.pointee.src_ptr = UnsafePointer(sourceBuffer)
+        stream.pointee.src_size = 0
+        stream.pointee.dst_ptr = destinationBuffer
+        stream.pointee.dst_size = chunkSize
+
+        var hasher = SHA256()
+        var runningCRC: uLong = crc32(0, nil, 0)
+        var totalDecompressedSize: UInt64 = 0
+        var inputExhausted = false
+
+        while status != COMPRESSION_STATUS_END {
+            if stream.pointee.src_size == 0 && !inputExhausted {
+                let toRead = min(chunkSize, compressedBytesRemaining)
+                if toRead > 0,
+                    let chunk = fileHandle.readData(ofLength: toRead) as Data?,
+                    !chunk.isEmpty
+                {
+                    compressedBytesRemaining -= chunk.count
+                    chunk.copyBytes(to: sourceBuffer, count: chunk.count)
+                    stream.pointee.src_ptr = UnsafePointer(sourceBuffer)
+                    stream.pointee.src_size = chunk.count
+                } else {
+                    inputExhausted = true
+                }
+            }
+
+            stream.pointee.dst_ptr = destinationBuffer
+            stream.pointee.dst_size = chunkSize
+
+            let flags: Int32 = inputExhausted ? Int32(COMPRESSION_STREAM_FINALIZE.rawValue) : 0
+            status = compression_stream_process(stream, flags)
+
+            switch status {
+            case COMPRESSION_STATUS_OK, COMPRESSION_STATUS_END:
+                let produced = chunkSize - stream.pointee.dst_size
+                if produced > 0 {
+                    let buf = UnsafeBufferPointer(start: destinationBuffer, count: produced)
+                    hasher.update(bufferPointer: UnsafeRawBufferPointer(buf))
+                    runningCRC = crc32(runningCRC, destinationBuffer, uInt(produced))
+                    totalDecompressedSize += UInt64(produced)
+                }
+            default:
+                throw ContainerizationError(.internalError, message: "gzip decompression failed")
+            }
+        }
+
+        let actualCRC = UInt32(truncatingIfNeeded: runningCRC)
+        let actualSize = UInt32(truncatingIfNeeded: totalDecompressedSize)
+
+        guard expectedCRC == actualCRC, expectedSize == actualSize else {
+            throw ContainerizationError(.internalError, message: "gzip trailer mismatch")
+        }
+
+        return hasher.finalize()
+    }
+
+    private static func readExactly(fileHandle: FileHandle, count: Int) -> Data? {
+        let data = fileHandle.readData(ofLength: count)
+        return data.isEmpty ? nil : data
+    }
+
+    private static func gzipHeaderSize(_ data: Data) throws -> Int {
+        guard data.count >= 10,
+            data[data.startIndex] == 0x1f,
+            data[data.startIndex + 1] == 0x8b,
+            data[data.startIndex + 2] == 0x08
+        else {
+            throw ContainerizationError(.internalError, message: "invalid gzip file")
+        }
+
+        let start = data.startIndex
+        let flags = data[start + 3]
+        var offset = 10
+
+        if flags & 0x04 != 0 {
+            guard data.count >= offset + 2 else {
+                throw ContainerizationError(.internalError, message: "invalid gzip file")
+            }
+            let extraLen = Int(data[start + offset]) | (Int(data[start + offset + 1]) << 8)
+            offset += 2 + extraLen
+        }
+        if flags & 0x08 != 0 {
+            while offset < data.count && data[start + offset] != 0 { offset += 1 }
+            offset += 1
+        }
+        if flags & 0x10 != 0 {
+            while offset < data.count && data[start + offset] != 0 { offset += 1 }
+            offset += 1
+        }
+        if flags & 0x02 != 0 { offset += 2 }
+
+        guard offset < data.count else {
+            throw ContainerizationError(.internalError, message: "invalid gzip file")
+        }
+        return offset
+    }
 }

Tests/ContainerizationOCITests/DiffIDTests.swift

@@ -0,0 +1,160 @@
+//===----------------------------------------------------------------------===//
+// Copyright © 2026 Apple Inc. and the Containerization project authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//===----------------------------------------------------------------------===//
+
+import ContainerizationError
+import Crypto
+import Foundation
+import Testing
+
+@testable import ContainerizationOCI
+
+struct DiffIDTests {
+    /// Helper to create a gzip-compressed temporary file from raw data.
+    private func createGzipFile(content: Data) throws -> URL {
+        let tempDir = FileManager.default.temporaryDirectory
+        let rawFile = tempDir.appendingPathComponent(UUID().uuidString)
+        let gzFile = tempDir.appendingPathComponent(UUID().uuidString + ".gz")
+        try content.write(to: rawFile)
+        defer { try? FileManager.default.removeItem(at: rawFile) }
+
+        let process = Process()
+        process.executableURL = URL(fileURLWithPath: "/usr/bin/gzip")
+        process.arguments = ["-k", "-f", rawFile.path]
+        try process.run()
+        process.waitUntilExit()
+
+        let gzPath = URL(fileURLWithPath: rawFile.path + ".gz")
+        if FileManager.default.fileExists(atPath: gzPath.path) {
+            try FileManager.default.moveItem(at: gzPath, to: gzFile)
+        }
+        return gzFile
+    }
+
+    @Test func diffIDMatchesUncompressedSHA256() throws {
+        let content = Data("hello, oci layer content for diffid test".utf8)
+        let gzFile = try createGzipFile(content: content)
+        defer { try? FileManager.default.removeItem(at: gzFile) }
+
+        let diffID = try ContentWriter.diffID(of: gzFile)
+        let expected = SHA256.hash(data: content)
+
+        #expect(diffID.digestString == expected.digestString)
+    }
+
+    @Test func diffIDIsDeterministic() throws {
+        let content = Data("deterministic diffid check".utf8)
+        let gzFile = try createGzipFile(content: content)
+        defer { try? FileManager.default.removeItem(at: gzFile) }
+
+        let first = try ContentWriter.diffID(of: gzFile)
+        let second = try ContentWriter.diffID(of: gzFile)
+
+        #expect(first.digestString == second.digestString)
+    }
+
+    @Test func diffIDRejectsNonGzipData() throws {
+        let tempFile = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+        try Data("this is not gzip".utf8).write(to: tempFile)
+        defer { try? FileManager.default.removeItem(at: tempFile) }
+
+        #expect(throws: ContainerizationError.self) {
+            try ContentWriter.diffID(of: tempFile)
+        }
+    }
+
+    @Test func diffIDRejectsEmptyFile() throws {
+        let tempFile = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString)
+        try Data().write(to: tempFile)
+        defer { try? FileManager.default.removeItem(at: tempFile) }
+
+        #expect(throws: ContainerizationError.self) {
+            try ContentWriter.diffID(of: tempFile)
+        }
+    }
+
+    @Test func diffIDHandlesLargeContent() throws {
+        // 1MB of repeating data
+        let pattern = Data("ABCDEFGHIJKLMNOPQRSTUVWXYZ012345".utf8)
+        var large = Data()
+        for _ in 0..<(1_048_576 / pattern.count) {
+            large.append(pattern)
+        }
+        let gzFile = try createGzipFile(content: large)
+        defer { try? FileManager.default.removeItem(at: gzFile) }
+
+        let diffID = try ContentWriter.diffID(of: gzFile)
+        let expected = SHA256.hash(data: large)
+
+        #expect(diffID.digestString == expected.digestString)
+    }
+
+    @Test func diffIDRejectsTruncatedGzip() throws {
+        // Build a valid gzip file, then chop off the 8-byte trailer (CRC32 + ISIZE)
+        // to produce a structurally malformed archive.
+        let content = Data("truncated gzip trailer test".utf8)
+        let gzFile = try createGzipFile(content: content)
+        defer { try? FileManager.default.removeItem(at: gzFile) }
+
+        var gzData = try Data(contentsOf: gzFile)
+        guard gzData.count > 8 else {
+            Issue.record("Compressed file too small to truncate")
+            return
+        }
+        gzData.removeLast(8)
+
+        let truncatedFile = FileManager.default.temporaryDirectory
+            .appendingPathComponent(UUID().uuidString + ".gz")
+        try gzData.write(to: truncatedFile)
+        defer { try? FileManager.default.removeItem(at: truncatedFile) }
+
+        #expect(throws: ContainerizationError.self) {
+            try ContentWriter.diffID(of: truncatedFile)
+        }
+    }
+
+    @Test func diffIDRejectsCorruptedCRC() throws {
+        // Flip a byte in the CRC32 field of an otherwise valid gzip file.
+        let content = Data("corrupted crc test".utf8)
+        let gzFile = try createGzipFile(content: content)
+        defer { try? FileManager.default.removeItem(at: gzFile) }
+
+        var gzData = try Data(contentsOf: gzFile)
+        let crcOffset = gzData.count - 8
+        gzData[crcOffset] ^= 0xFF
+
+        let corruptedFile = FileManager.default.temporaryDirectory
+            .appendingPathComponent(UUID().uuidString + ".gz")
+        try gzData.write(to: corruptedFile)
+        defer { try? FileManager.default.removeItem(at: corruptedFile) }
+
+        #expect(throws: ContainerizationError.self) {
+            try ContentWriter.diffID(of: corruptedFile)
+        }
+    }
+
+    @Test func diffIDDigestStringFormat() throws {
+        let content = Data("format test".utf8)
+        let gzFile = try createGzipFile(content: content)
+        defer { try? FileManager.default.removeItem(at: gzFile) }
+
+        let diffID = try ContentWriter.diffID(of: gzFile)
+        let digestString = diffID.digestString
+
+        #expect(digestString.hasPrefix("sha256:"))
+        // sha256: prefix + 64 hex chars
+        #expect(digestString.count == 7 + 64)
+    }
+}