Skip to content

Use native, leaveOpen tar streaming to decompress the tar without copy #54757

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
nagilson wants to merge 1 commit into
base: release/dnup
Choose a base branch
from
Draft

Use native, leaveOpen tar streaming to decompress the tar without copy #54757

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 63 additions & 54 deletions src/Installer/Microsoft.Dotnet.Installation/Internal/DotnetArchiveExtractor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,57 +303,51 @@ private static string ResolveEntryDestPath(string entryName, string targetDir, M
/// </summary>
private static void ExtractTarArchive(string archivePath, string targetDir, IProgressTask? installTask, MuxerHandler? muxerHandler = null, Action<string>? onEntryExtracted = null, Func<string, bool>? shouldSkipEntry = null)
{
string decompressedPath = DecompressTarGzIfNeeded(archivePath, out bool needsDecompression);
bool isGzip = archivePath.EndsWith(".gz", StringComparison.OrdinalIgnoreCase);

try
{
long totalEntries = CountTarEntries(decompressedPath);
installTask?.MaxValue = totalEntries > 0 ? totalEntries : 1;
// Hold a single read handle on the archive for the entire extraction. FileShare.Read
// keeps the file locked against deletion on Windows, and on Unix an open handle keeps the
// underlying data readable even if the path is unlinked. Either way the scratch directory
// cannot be reaped out from under us (e.g. by CI temp cleanup) in the window between the
// entry-count pass and the extraction pass. Both passes seek this shared stream back to the
// start and wrap it in a fresh, non-owning GZipStream/TarReader so the handle stays open.
using var archiveStream = new FileStream(archivePath, FileMode.Open, FileAccess.Read, FileShare.Read);

ExtractTarContents(decompressedPath, targetDir, installTask, muxerHandler, onEntryExtracted, shouldSkipEntry);
}
finally
{
// Clean up temporary decompressed file
if (needsDecompression && File.Exists(decompressedPath))
{
File.Delete(decompressedPath);
}
}
long totalEntries = CountTarEntries(archiveStream, isGzip);
installTask?.MaxValue = totalEntries > 0 ? totalEntries : 1;

archiveStream.Seek(0, SeekOrigin.Begin);
ExtractTarContents(archiveStream, isGzip, targetDir, installTask, muxerHandler, onEntryExtracted, shouldSkipEntry);
}

/// <summary>
/// Decompresses a .tar.gz file if needed, returning the path to the tar file.
/// Wraps an already-open tar archive stream for reading, layering in gzip decompression when
/// needed. The returned reader leaves <paramref name="archiveStream"/> open so a single file
/// handle can be reused across multiple passes; callers dispose the returned stream only when
/// it is a gzip wrapper (never the shared archive stream).
/// </summary>
private static string DecompressTarGzIfNeeded(string archivePath, out bool needsDecompression)
{
needsDecompression = archivePath.EndsWith(".gz", StringComparison.OrdinalIgnoreCase);
if (!needsDecompression)
{
return archivePath;
}

string decompressedPath = archivePath.Replace(".gz", "", StringComparison.Ordinal);

using FileStream originalFileStream = File.OpenRead(archivePath);
using FileStream decompressedFileStream = File.Create(decompressedPath);
using GZipStream decompressionStream = new GZipStream(originalFileStream, CompressionMode.Decompress);
decompressionStream.CopyTo(decompressedFileStream);

return decompressedPath;
}
private static Stream OpenTarReadStream(Stream archiveStream, bool isGzip)
=> isGzip ? new GZipStream(archiveStream, CompressionMode.Decompress, leaveOpen: true) : archiveStream;

/// <summary>
/// Counts the number of entries in a tar file for progress reporting.
/// Counts the number of entries in a tar archive for progress reporting.
/// </summary>
private static long CountTarEntries(string tarPath)
private static long CountTarEntries(Stream archiveStream, bool isGzip)
{
long totalFiles = 0;
using var tarStream = File.OpenRead(tarPath);
var tarReader = new TarReader(tarStream);
while (tarReader.GetNextEntry() is not null)
Stream tarStream = OpenTarReadStream(archiveStream, isGzip);
try
{
totalFiles++;
using var tarReader = new TarReader(tarStream, leaveOpen: true);
while (tarReader.GetNextEntry() is not null)
{
totalFiles++;
}
}
finally
{
// Only dispose the gzip wrapper; the shared archive stream stays open for the next pass.
if (isGzip) { tarStream.Dispose(); }
}
return totalFiles;
}
Expand All @@ -364,27 +358,42 @@ private static long CountTarEntries(string tarPath)
/// </summary>
internal static void ExtractTarContents(string tarPath, string targetDir, IProgressTask? installTask, MuxerHandler? muxerHandler = null, Action<string>? onEntryExtracted = null, Func<string, bool>? shouldSkipEntry = null)
{
using var tarStream = File.OpenRead(tarPath);
var tarReader = new TarReader(tarStream);
TarEntry? entry;

// Defer hard link creation until after all regular files are extracted,
// since the target file may not exist yet when the hard link entry is encountered.
var deferredHardLinks = new List<(string DestPath, string TargetPath)>();
bool isGzip = tarPath.EndsWith(".gz", StringComparison.OrdinalIgnoreCase);
using var archiveStream = new FileStream(tarPath, FileMode.Open, FileAccess.Read, FileShare.Read);
ExtractTarContents(archiveStream, isGzip, targetDir, installTask, muxerHandler, onEntryExtracted, shouldSkipEntry);
}

while ((entry = tarReader.GetNextEntry()) is not null)
private static void ExtractTarContents(Stream archiveStream, bool isGzip, string targetDir, IProgressTask? installTask, MuxerHandler? muxerHandler, Action<string>? onEntryExtracted, Func<string, bool>? shouldSkipEntry)
{
Stream tarStream = OpenTarReadStream(archiveStream, isGzip);
try
{
bool skip = shouldSkipEntry?.Invoke(entry.Name) ?? false;
if (!skip)
using var tarReader = new TarReader(tarStream, leaveOpen: true);
TarEntry? entry;

// Defer hard link creation until after all regular files are extracted,
// since the target file may not exist yet when the hard link entry is encountered.
var deferredHardLinks = new List<(string DestPath, string TargetPath)>();

while ((entry = tarReader.GetNextEntry()) is not null)
{
ProcessTarEntry(entry, targetDir, muxerHandler, deferredHardLinks);
bool skip = shouldSkipEntry?.Invoke(entry.Name) ?? false;
if (!skip)
{
ProcessTarEntry(entry, targetDir, muxerHandler, deferredHardLinks);
}

onEntryExtracted?.Invoke(entry.Name);
installTask?.Value += 1;
}

onEntryExtracted?.Invoke(entry.Name);
installTask?.Value += 1;
CreateDeferredHardLinks(deferredHardLinks);
}
finally
{
// Only dispose the gzip wrapper; the shared archive stream is owned by the caller.
if (isGzip) { tarStream.Dispose(); }
}

CreateDeferredHardLinks(deferredHardLinks);
}

private static void ProcessTarEntry(TarEntry entry, string targetDir, MuxerHandler? muxerHandler, List<(string DestPath, string TargetPath)> deferredHardLinks)
Expand Down
39 changes: 39 additions & 0 deletions test/dotnetup.Tests/DotnetArchiveExtractorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,45 @@ public void Commit_ExtractsTarGzArchive_Correctly()
File.ReadAllText(runtimeFile).Should().Be("runtime-content");
}

[Fact]
public void Commit_ExtractsTarGzArchive_WhenDecompressedTarPathAlreadyExists()
{
using var testEnv = DotnetupTestUtilities.CreateTestEnvironment();

var installRoot = new DotnetInstallRoot(testEnv.InstallPath, InstallerUtilities.GetDefaultInstallArchitecture());
var version = new ReleaseVersion(9, 0, 0);

var request = new DotnetInstallRequest(
installRoot,
new UpdateChannel("9.0"),
InstallComponent.Runtime,
new InstallRequestOptions());

byte[] tarGzContent = CreateTarGzWithFiles(
("shared/Microsoft.NETCore.App/9.0.0/System.Runtime.dll", "runtime-content"));

var mockDownloader = new MockArchiveDownloader
{
CreateFakeArchive = true,
FakeArchiveContent = tarGzContent,
ArchiveFileExtension = ".tar.gz"
};

var releaseManifest = new ReleaseManifest();
var progressTarget = new NullProgressTarget();

using var extractor = new DotnetArchiveExtractor(request, version, releaseManifest, progressTarget, mockDownloader);

extractor.Prepare();
Directory.CreateDirectory(mockDownloader.DownloadCalls[0].DestinationPath.Replace(".gz", "", StringComparison.Ordinal));

extractor.Commit();

var runtimeFile = Path.Combine(testEnv.InstallPath, "shared", "Microsoft.NETCore.App", "9.0.0", "System.Runtime.dll");
File.Exists(runtimeFile).Should().BeTrue("tar.gz extraction should not depend on a sibling decompressed .tar staging file");
File.ReadAllText(runtimeFile).Should().Be("runtime-content");
}

[PlatformSpecificFact(TestPlatforms.Windows)]
public void Commit_ExtractsZipArchive_Correctly()
{
Expand Down
Loading