diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8642c83eb1..9ade24494e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -815,6 +815,31 @@ jobs: -p:Version=${{ steps.version.outputs.version }} --output nupkg + # NuGet's OPC writer gives the core-properties *.psmdcp part a random + # name on each pack run. Normalize that internal path before hashing or + # publishing so otherwise identical .nupkg/.snupkg archives compare + # byte-for-byte. See issue #2756 / DEVELOPER_GUIDE.md. + # NuGet の OPC writer は core-properties の *.psmdcp part に pack ごと + # ランダムな名前を付ける。hash / publish 前に内部 path を正規化し、 + # それ以外が同一の .nupkg/.snupkg archive を byte-for-byte で比較可能にする。 + # 詳細は issue #2756 / DEVELOPER_GUIDE.md。 + - name: Normalize NuGet package metadata part names + run: dotnet run --project tools/CodeIndex.PackageNormalize -- nupkg/*.nupkg nupkg/*.snupkg + + - name: Verify normalized NuGet package metadata part names + run: | + set -euo pipefail + for package in nupkg/*.nupkg nupkg/*.snupkg; do + unzip -Z1 "$package" | grep -qx 'package/services/metadata/core-properties/core-properties.psmdcp' + if unzip -Z1 "$package" \ + | grep '^package/services/metadata/core-properties/' \ + | grep -vx 'package/services/metadata/core-properties/core-properties.psmdcp'; then + echo "Unexpected non-canonical NuGet core-properties part in ${package}." >&2 + unzip -Z1 "$package" >&2 + exit 1 + fi + done + - name: Verify packed NuGet package version env: VERSION: ${{ steps.version.outputs.version }} diff --git a/CodeIndex.sln b/CodeIndex.sln index 24e21b4aed..579ab2b8d9 100644 --- a/CodeIndex.sln +++ b/CodeIndex.sln @@ -15,6 +15,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodeIndex.Changelog", "tool EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodeIndex.TestTelemetry", "tools\CodeIndex.TestTelemetry\CodeIndex.TestTelemetry.csproj", "{4EA94432-F249-46FC-8EB4-99F1EB9933A0}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodeIndex.PackageNormalize", "tools\CodeIndex.PackageNormalize\CodeIndex.PackageNormalize.csproj", "{1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -73,6 +75,18 @@ Global {4EA94432-F249-46FC-8EB4-99F1EB9933A0}.Release|x64.Build.0 = Release|Any CPU {4EA94432-F249-46FC-8EB4-99F1EB9933A0}.Release|x86.ActiveCfg = Release|Any CPU {4EA94432-F249-46FC-8EB4-99F1EB9933A0}.Release|x86.Build.0 = Release|Any CPU + {1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4}.Debug|x64.ActiveCfg = Debug|Any CPU + {1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4}.Debug|x64.Build.0 = Debug|Any CPU + {1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4}.Debug|x86.ActiveCfg = Debug|Any CPU + {1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4}.Debug|x86.Build.0 = Debug|Any CPU + {1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4}.Release|Any CPU.Build.0 = Release|Any CPU + {1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4}.Release|x64.ActiveCfg = Release|Any CPU + {1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4}.Release|x64.Build.0 = Release|Any CPU + {1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4}.Release|x86.ActiveCfg = Release|Any CPU + {1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -82,5 +96,6 @@ Global {A33D0351-ACAC-485B-9517-9ABD6A878ECD} = {0AB3BF05-4346-4AA6-1389-037BE0695223} {6A1D4E0B-24F8-4E8E-9E7A-28CC55D3F7AC} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {4EA94432-F249-46FC-8EB4-99F1EB9933A0} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {1B9BA505-A3AE-4F70-9E7B-18EB4D3540E4} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} EndGlobalSection EndGlobal diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index cfb1f1838b..7ef4a341c2 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -151,6 +151,16 @@ The normal build/test workflow also runs `dotnet list src/CodeIndex/CodeIndex.cs The release `dotnet publish` (per-RID) and `dotnet pack` (NuGet packaging) steps intentionally do **not** set `RestoreLockedMode=true`. Those steps run runtime-specific restores that legitimately add lock entries that did not exist at solution-restore time (e.g. `net8.0/` runtime sections, `Microsoft.NET.ILLink.Tasks` for trimming). They still consume locked versions because `RestorePackagesWithLockFile=true` from `Directory.Build.props` forces every restore on the machine to resolve through the lock file. The supply-chain guarantee for `Microsoft.Data.Sqlite` and its `SQLitePCLRaw.*` graph is enforced by the solution-level locked restore that runs first. +After `dotnet pack`, the release workflow runs +`dotnet run --project tools/CodeIndex.PackageNormalize -- nupkg/*.nupkg nupkg/*.snupkg` +before validating, hashing, or publishing NuGet artifacts. NuGet's OPC package +writer generates a random `package/services/metadata/core-properties/*.psmdcp` +part name on each pack run; the normalizer rewrites that part to +`package/services/metadata/core-properties/core-properties.psmdcp`, updates the +matching content-type and relationship references, and gives ZIP entries stable +timestamps. This is the package reproducibility boundary for `.nupkg` and +`.snupkg` archives (#2756). + When you intentionally update a dependency (or add a new direct `PackageReference`), regenerate the lock files locally and commit the diff in the same change: ```bash @@ -2030,6 +2040,15 @@ CI（`.github/workflows/dotnet.yml`, `release.yml`, `codeql.yml`）はソリュ release の `dotnet publish`（RID ごと）と `dotnet pack`（NuGet パッケージング）には意図的に `RestoreLockedMode=true` を設定していません。これらは runtime-specific な restore を走らせ、ソリューション restore 時には存在しなかった lock エントリ（`net8.0/` 等の runtime section や trimming 用の `Microsoft.NET.ILLink.Tasks`）を正当に追加します。それでも `Directory.Build.props` の `RestorePackagesWithLockFile=true` により、その実行マシン上の全 restore は lock ファイル経由で解決されるため版は固定されたままです。`Microsoft.Data.Sqlite` および `SQLitePCLRaw.*` グラフに対する supply-chain 保証は、先行する solution-level の locked restore で担保されます。 +`dotnet pack` 後、release workflow は NuGet artifact の検証、hash、publish の前に +`dotnet run --project tools/CodeIndex.PackageNormalize -- nupkg/*.nupkg nupkg/*.snupkg` +を実行します。NuGet の OPC package writer は +`package/services/metadata/core-properties/*.psmdcp` part 名を pack ごとに +ランダム生成するため、normalizer はその part を +`package/services/metadata/core-properties/core-properties.psmdcp` に書き換え、 +対応する content-type / relationship 参照も更新し、ZIP entry timestamp を固定します。 +これが `.nupkg` / `.snupkg` archive の package 再現性境界です (#2756)。 + 依存を意図的に更新する（あるいは直接 `PackageReference` を追加する）場合は、ローカルで lock ファイルを再生成し、同じ変更でコミットしてください: ```bash diff --git a/changelog.d/unreleased/2756.fixed.md b/changelog.d/unreleased/2756.fixed.md new file mode 100644 index 0000000000..311f89fe62 --- /dev/null +++ b/changelog.d/unreleased/2756.fixed.md @@ -0,0 +1,18 @@ +--- +category: fixed +issues: + - 2756 +affected: + - .github/workflows/release.yml + - DEVELOPER_GUIDE.md + - tools/CodeIndex.PackageNormalize/PackageNormalizeCli.cs + - tests/CodeIndex.Tests/ReleaseWorkflowTests.cs +--- + +## English + +- **NuGet release packages now normalize the random core-properties part name (#2756)** — the release workflow rewrites `.nupkg` and `.snupkg` archives so NuGet's generated `package/services/metadata/core-properties/*.psmdcp` entry uses a stable path before package validation, hashing, and publishing. + +## 日本語 + +- **NuGet release package のランダムな core-properties part 名を正規化するようになりました (#2756)** — release workflow は package 検証、hash、publish の前に `.nupkg` / `.snupkg` archive を書き換え、NuGet が生成する `package/services/metadata/core-properties/*.psmdcp` entry を安定した path に固定します。 diff --git a/tests/CodeIndex.Tests/CodeIndex.Tests.csproj b/tests/CodeIndex.Tests/CodeIndex.Tests.csproj index 5eb4183262..77843d7191 100644 --- a/tests/CodeIndex.Tests/CodeIndex.Tests.csproj +++ b/tests/CodeIndex.Tests/CodeIndex.Tests.csproj @@ -27,6 +27,7 @@ + diff --git a/tests/CodeIndex.Tests/ReleaseWorkflowTests.cs b/tests/CodeIndex.Tests/ReleaseWorkflowTests.cs index c048916878..294fd4a554 100644 --- a/tests/CodeIndex.Tests/ReleaseWorkflowTests.cs +++ b/tests/CodeIndex.Tests/ReleaseWorkflowTests.cs @@ -1,3 +1,8 @@ +using CodeIndex.PackageNormalize; +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; + namespace CodeIndex.Tests; public class ReleaseWorkflowTests @@ -89,6 +94,58 @@ public void ReleaseWorkflow_ValidatesNuGetVersionBeforePublishing() Assert.DoesNotContain("--skip-duplicate", workflow); } + // Issue #2756: NuGet emits the core-properties OPC part with a random + // *.psmdcp entry name, so two otherwise identical pack runs can produce + // different .nupkg/.snupkg bytes. The release workflow normalizes that + // implementation detail before hashing and publishing. + // Issue #2756 対応: NuGet は core-properties の OPC part をランダムな + // *.psmdcp entry 名で生成するため、他が同一でも .nupkg/.snupkg の bytes が + // 揺れる。release workflow は hash / publish 前にその実装詳細を正規化する。 + [Fact] + public void ReleaseWorkflow_NormalizesNuGetCorePropertiesBeforePublishing() + { + var workflow = File.ReadAllText(Path.Combine(GetRepositoryRoot(), ".github", "workflows", "release.yml")); + + Assert.Contains("Normalize NuGet package metadata part names", workflow); + Assert.Contains("dotnet run --project tools/CodeIndex.PackageNormalize --", workflow); + Assert.Contains("nupkg/*.nupkg nupkg/*.snupkg", workflow); + Assert.Contains("core-properties/core-properties.psmdcp", workflow); + } + + [Fact] + public void PackageNormalizer_RewritesRandomCorePropertiesPartDeterministically() + { + var projectRoot = TestProjectHelper.CreateTempProject(nameof(PackageNormalizer_RewritesRandomCorePropertiesPartDeterministically)); + try + { + var packageA = Path.Combine(projectRoot, "a.nupkg"); + var packageB = Path.Combine(projectRoot, "b.nupkg"); + + CreateMinimalNuGetPackage(packageA, "a1b2c3.psmdcp"); + CreateMinimalNuGetPackage(packageB, "f9e8d7.psmdcp"); + + PackageCorePropertiesNormalizer.NormalizePackage(packageA); + PackageCorePropertiesNormalizer.NormalizePackage(packageB); + + Assert.Equal( + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(packageA))), + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(packageB)))); + + using var archive = ZipFile.OpenRead(packageA); + Assert.Contains(archive.Entries, entry => entry.FullName == PackageCorePropertiesNormalizer.CanonicalCorePropertiesPath); + Assert.DoesNotContain(archive.Entries, entry => entry.FullName.EndsWith("a1b2c3.psmdcp", StringComparison.Ordinal)); + + var contentTypes = ReadZipEntryText(archive, "[Content_Types].xml"); + var relationships = ReadZipEntryText(archive, "_rels/.rels"); + Assert.Contains("/package/services/metadata/core-properties/core-properties.psmdcp", contentTypes); + Assert.Contains("/package/services/metadata/core-properties/core-properties.psmdcp", relationships); + } + finally + { + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + [Fact] public void ReleaseWorkflow_PublishesOfficialContainerImage() { @@ -124,4 +181,50 @@ private static string GetRepositoryRoot() throw new InvalidOperationException("Could not locate repository root / リポジトリルートを特定できませんでした"); } + + private static void CreateMinimalNuGetPackage(string packagePath, string corePropertiesFileName) + { + var corePropertiesPath = $"package/services/metadata/core-properties/{corePropertiesFileName}"; + using var archive = ZipFile.Open(packagePath, ZipArchiveMode.Create); + + WriteZipEntry(archive, "[Content_Types].xml", $""" + + + + + + """); + WriteZipEntry(archive, "_rels/.rels", $""" + + + + + """); + WriteZipEntry(archive, "cdidx.nuspec", """ + + + cdidx1.0.0 + + """); + WriteZipEntry(archive, corePropertiesPath, """ + + + """); + } + + private static void WriteZipEntry(ZipArchive archive, string entryName, string content) + { + var entry = archive.CreateEntry(entryName); + using var stream = entry.Open(); + using var writer = new StreamWriter(stream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + writer.Write(content.Replace("\r\n", "\n", StringComparison.Ordinal)); + } + + private static string ReadZipEntryText(ZipArchive archive, string entryName) + { + var entry = archive.GetEntry(entryName) ?? throw new InvalidOperationException($"Missing ZIP entry: {entryName}"); + using var stream = entry.Open(); + using var reader = new StreamReader(stream, Encoding.UTF8); + return reader.ReadToEnd(); + } } diff --git a/tests/CodeIndex.Tests/packages.lock.json b/tests/CodeIndex.Tests/packages.lock.json index 6ea4e3d793..d42e936e32 100644 --- a/tests/CodeIndex.Tests/packages.lock.json +++ b/tests/CodeIndex.Tests/packages.lock.json @@ -1207,6 +1207,9 @@ "codeindex.changelog": { "type": "Project" }, + "codeindex.packagenormalize": { + "type": "Project" + }, "codeindex.testtelemetry": { "type": "Project" } @@ -2417,9 +2420,12 @@ "codeindex.changelog": { "type": "Project" }, + "codeindex.packagenormalize": { + "type": "Project" + }, "codeindex.testtelemetry": { "type": "Project" } } } -} +} \ No newline at end of file diff --git a/tools/CodeIndex.PackageNormalize/CodeIndex.PackageNormalize.csproj b/tools/CodeIndex.PackageNormalize/CodeIndex.PackageNormalize.csproj new file mode 100644 index 0000000000..2b8d4aed18 --- /dev/null +++ b/tools/CodeIndex.PackageNormalize/CodeIndex.PackageNormalize.csproj @@ -0,0 +1,11 @@ + + + + Exe + net8.0 + enable + enable + false + + + diff --git a/tools/CodeIndex.PackageNormalize/PackageNormalizeCli.cs b/tools/CodeIndex.PackageNormalize/PackageNormalizeCli.cs new file mode 100644 index 0000000000..b8dca1d0c0 --- /dev/null +++ b/tools/CodeIndex.PackageNormalize/PackageNormalizeCli.cs @@ -0,0 +1,107 @@ +using System.IO.Compression; +using System.Text; + +namespace CodeIndex.PackageNormalize; + +public static class PackageNormalizeCli +{ + public static int Run(string[] args) + { + if (args.Length == 0 || args.Any(arg => arg is "-h" or "--help")) + { + Console.Error.WriteLine("Usage: dotnet run --project tools/CodeIndex.PackageNormalize -- [...]"); + return args.Length == 0 ? 1 : 0; + } + + foreach (var packagePath in args) + { + PackageCorePropertiesNormalizer.NormalizePackage(packagePath); + Console.WriteLine($"Normalized {packagePath}"); + } + + return 0; + } +} + +public static class PackageCorePropertiesNormalizer +{ + public const string CanonicalCorePropertiesPath = "package/services/metadata/core-properties/core-properties.psmdcp"; + + private static readonly DateTimeOffset StableZipTimestamp = new(1980, 1, 1, 0, 0, 0, TimeSpan.Zero); + + public static void NormalizePackage(string packagePath) + { + ArgumentException.ThrowIfNullOrWhiteSpace(packagePath); + + var fullPath = Path.GetFullPath(packagePath); + var tempPath = fullPath + ".normalize-tmp"; + if (File.Exists(tempPath)) + File.Delete(tempPath); + + using (var sourceStream = File.Open(fullPath, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (var sourceArchive = new ZipArchive(sourceStream, ZipArchiveMode.Read, leaveOpen: false)) + using (var destinationStream = File.Open(tempPath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None)) + using (var destinationArchive = new ZipArchive(destinationStream, ZipArchiveMode.Create, leaveOpen: false)) + { + var corePropertiesEntries = sourceArchive.Entries + .Where(entry => IsCorePropertiesPart(entry.FullName)) + .ToArray(); + + if (corePropertiesEntries.Length != 1) + throw new InvalidOperationException($"Expected exactly one NuGet core-properties part in {packagePath}, found {corePropertiesEntries.Length}."); + + var originalCorePropertiesPath = corePropertiesEntries[0].FullName; + var usedNames = new HashSet(StringComparer.Ordinal); + + foreach (var sourceEntry in sourceArchive.Entries) + { + var destinationName = sourceEntry.FullName == originalCorePropertiesPath + ? CanonicalCorePropertiesPath + : sourceEntry.FullName; + + if (!usedNames.Add(destinationName)) + throw new InvalidOperationException($"Duplicate ZIP entry after normalization: {destinationName}"); + + var destinationEntry = destinationArchive.CreateEntry(destinationName, CompressionLevel.Optimal); + destinationEntry.LastWriteTime = StableZipTimestamp; + destinationEntry.ExternalAttributes = sourceEntry.ExternalAttributes; + + using var sourceEntryStream = sourceEntry.Open(); + using var destinationEntryStream = destinationEntry.Open(); + + if (NeedsXmlReferenceRewrite(sourceEntry.FullName)) + { + using var reader = new StreamReader(sourceEntryStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: false); + using var writer = new StreamWriter(destinationEntryStream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), leaveOpen: false); + writer.Write(RewriteCorePropertiesReferences(reader.ReadToEnd(), originalCorePropertiesPath)); + } + else + { + sourceEntryStream.CopyTo(destinationEntryStream); + } + } + } + + File.Move(tempPath, fullPath, overwrite: true); + } + + private static bool IsCorePropertiesPart(string entryName) + { + return entryName.StartsWith("package/services/metadata/core-properties/", StringComparison.Ordinal) + && entryName.EndsWith(".psmdcp", StringComparison.Ordinal); + } + + private static bool NeedsXmlReferenceRewrite(string entryName) + { + return entryName.Equals("[Content_Types].xml", StringComparison.Ordinal) + || entryName.EndsWith(".rels", StringComparison.Ordinal); + } + + private static string RewriteCorePropertiesReferences(string content, string originalCorePropertiesPath) + { + var canonical = CanonicalCorePropertiesPath; + return content + .Replace(originalCorePropertiesPath, canonical, StringComparison.Ordinal) + .Replace("/" + originalCorePropertiesPath, "/" + canonical, StringComparison.Ordinal); + } +} diff --git a/tools/CodeIndex.PackageNormalize/Program.cs b/tools/CodeIndex.PackageNormalize/Program.cs new file mode 100644 index 0000000000..2ca7333847 --- /dev/null +++ b/tools/CodeIndex.PackageNormalize/Program.cs @@ -0,0 +1,3 @@ +using CodeIndex.PackageNormalize; + +return PackageNormalizeCli.Run(args); diff --git a/tools/CodeIndex.PackageNormalize/packages.lock.json b/tools/CodeIndex.PackageNormalize/packages.lock.json new file mode 100644 index 0000000000..807ab822b6 --- /dev/null +++ b/tools/CodeIndex.PackageNormalize/packages.lock.json @@ -0,0 +1,6 @@ +{ + "version": 1, + "dependencies": { + "net8.0": {} + } +} \ No newline at end of file