From c770a6d1edaff7965e21f4e11274de0a9cb79690 Mon Sep 17 00:00:00 2001 From: Tom Longhurst <30480171+thomhurst@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:07:46 +0100 Subject: [PATCH] fix: retry transient image pull failures Bound retries to three attempts, preserve caller cancellation, and avoid retrying permanent Docker API responses. Refs #1733 --- .../Clients/DockerImageOperations.cs | 5 +- .../Clients/DockerImagePullRetryPolicy.cs | 93 +++++++ src/Testcontainers/Logging.cs | 8 + .../Clients/DockerImagePullRetryPolicyTest.cs | 261 ++++++++++++++++++ 4 files changed, 366 insertions(+), 1 deletion(-) create mode 100644 src/Testcontainers/Clients/DockerImagePullRetryPolicy.cs create mode 100644 tests/Testcontainers.Tests/Unit/Clients/DockerImagePullRetryPolicyTest.cs diff --git a/src/Testcontainers/Clients/DockerImageOperations.cs b/src/Testcontainers/Clients/DockerImageOperations.cs index e6f979614..c9b2480a8 100644 --- a/src/Testcontainers/Clients/DockerImageOperations.cs +++ b/src/Testcontainers/Clients/DockerImageOperations.cs @@ -70,7 +70,10 @@ public async Task CreateAsync(IImage image, IDockerRegistryAuthenticationConfigu IdentityToken = dockerRegistryAuthConfig.IdentityToken, }; - await DockerClient.Images.CreateImageAsync(createParameters, authConfig, traceProgress, ct) + await DockerImagePullRetryPolicy.ExecuteAsync( + token => DockerClient.Images.CreateImageAsync(createParameters, authConfig, traceProgress, token), + (attempt, delay, reason) => Logger.RetryDockerImagePull(image, attempt, DockerImagePullRetryPolicy.MaxAttempts, delay, reason), + ct) .ConfigureAwait(false); Logger.DockerImageCreated(image); diff --git a/src/Testcontainers/Clients/DockerImagePullRetryPolicy.cs b/src/Testcontainers/Clients/DockerImagePullRetryPolicy.cs new file mode 100644 index 000000000..423e529b9 --- /dev/null +++ b/src/Testcontainers/Clients/DockerImagePullRetryPolicy.cs @@ -0,0 +1,93 @@ +namespace DotNet.Testcontainers.Clients +{ + using System; + using System.IO; + using System.Net; + using System.Net.Http; + using System.Net.Sockets; + using System.Threading; + using System.Threading.Tasks; + using Docker.DotNet; + + internal static class DockerImagePullRetryPolicy + { + internal const int MaxAttempts = 3; + + private const int InitialDelayInMilliseconds = 1000; + + private static readonly Random Random = new Random(); + + public static Task ExecuteAsync(Func pull, Action onRetry, CancellationToken ct) + { + return ExecuteAsync(pull, onRetry, GetRetryDelay, (delay, token) => Task.Delay(delay, token), ct); + } + + internal static async Task ExecuteAsync(Func pull, Action onRetry, Func getRetryDelay, Func delay, CancellationToken ct) + { + for (var attempt = 1; ; attempt++) + { + ct.ThrowIfCancellationRequested(); + + try + { + await pull(ct) + .ConfigureAwait(false); + + return; + } + catch (Exception exception) when (attempt < MaxAttempts && IsTransient(exception, ct)) + { + var retryDelay = getRetryDelay(attempt); + onRetry(attempt + 1, retryDelay, GetFailureReason(exception)); + + await delay(retryDelay, ct) + .ConfigureAwait(false); + } + } + } + + internal static bool IsTransient(Exception exception, CancellationToken ct) + { + if (exception is DockerApiException dockerApiException) + { + var statusCode = (int)dockerApiException.StatusCode; + return HttpStatusCode.RequestTimeout.Equals(dockerApiException.StatusCode) + || (HttpStatusCode)429 == dockerApiException.StatusCode + || statusCode >= 500 && statusCode <= 599; + } + + if (exception is OperationCanceledException) + { + return !ct.IsCancellationRequested; + } + + return exception is HttpRequestException + || exception is IOException + || exception is SocketException + || exception is TimeoutException; + } + + internal static TimeSpan GetRetryDelay(int attempt) + { + var exponentialDelay = InitialDelayInMilliseconds * (1 << (attempt - 1)); + int jitter; + + lock (Random) + { + jitter = Random.Next(0, exponentialDelay / 4 + 1); + } + + return TimeSpan.FromMilliseconds(exponentialDelay + jitter); + } + + private static string GetFailureReason(Exception exception) + { + if (exception is DockerApiException dockerApiException) + { + return $"Docker API status code {(int)dockerApiException.StatusCode} ({dockerApiException.StatusCode})"; + } + + return exception.GetType().Name; + } + } +} diff --git a/src/Testcontainers/Logging.cs b/src/Testcontainers/Logging.cs index b4332bd5d..1a0a6f45e 100644 --- a/src/Testcontainers/Logging.cs +++ b/src/Testcontainers/Logging.cs @@ -69,6 +69,9 @@ internal static partial class Logging [LoggerMessage(Level = LogLevel.Information, Message = "Docker image {FullName} created")] private static partial void DockerImageCreatedCore(ILogger logger, string fullName); + [LoggerMessage(Level = LogLevel.Warning, Message = "Retry Docker image {FullName} pull, attempt {Attempt}/{MaxAttempts} in {Delay} due to {Reason}")] + private static partial void RetryDockerImagePullCore(ILogger logger, string fullName, int attempt, int maxAttempts, TimeSpan delay, string reason); + [LoggerMessage(Level = LogLevel.Information, Message = "Docker image {FullName} built")] private static partial void DockerImageBuiltCore(ILogger logger, string fullName); @@ -214,6 +217,11 @@ public static void DockerImageCreated(this ILogger logger, IImage image) DockerImageCreatedCore(logger, image.FullName); } + public static void RetryDockerImagePull(this ILogger logger, IImage image, int attempt, int maxAttempts, TimeSpan delay, string reason) + { + RetryDockerImagePullCore(logger, image.FullName, attempt, maxAttempts, delay, reason); + } + public static void DockerImageBuilt(this ILogger logger, IImage image) { DockerImageBuiltCore(logger, image.FullName); diff --git a/tests/Testcontainers.Tests/Unit/Clients/DockerImagePullRetryPolicyTest.cs b/tests/Testcontainers.Tests/Unit/Clients/DockerImagePullRetryPolicyTest.cs new file mode 100644 index 000000000..68fd031b9 --- /dev/null +++ b/tests/Testcontainers.Tests/Unit/Clients/DockerImagePullRetryPolicyTest.cs @@ -0,0 +1,261 @@ +namespace DotNet.Testcontainers.Tests.Unit +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Net.Sockets; + using System.Threading; + using System.Threading.Tasks; + using Docker.DotNet; + using DotNet.Testcontainers.Clients; + using Xunit; + + public sealed class DockerImagePullRetryPolicyTest + { + public static IEnumerable TransientExceptions { get; } = new Exception[] + { + new DockerApiException(HttpStatusCode.RequestTimeout, null), + new DockerApiException((HttpStatusCode)429, null), + new DockerApiException(HttpStatusCode.InternalServerError, null), + new DockerApiException(HttpStatusCode.NotImplemented, null), + new DockerApiException(HttpStatusCode.BadGateway, null), + new DockerApiException(HttpStatusCode.ServiceUnavailable, null), + new DockerApiException(HttpStatusCode.GatewayTimeout, null), + new DockerApiException((HttpStatusCode)599, null), + new HttpRequestException(), + new IOException(), + new SocketException(), + new TimeoutException(), + new TaskCanceledException(), + }.Select(exception => new object[] { exception }); + + public static IEnumerable PermanentExceptions { get; } = new Exception[] + { + new DockerApiException(HttpStatusCode.BadRequest, null), + new DockerApiException(HttpStatusCode.Unauthorized, null), + new DockerApiException(HttpStatusCode.Forbidden, null), + new DockerApiException(HttpStatusCode.NotFound, null), + new DockerApiException(HttpStatusCode.Conflict, null), + new DockerApiException((HttpStatusCode)422, null), + new DockerApiException((HttpStatusCode)600, null), + new InvalidOperationException(), + }.Select(exception => new object[] { exception }); + + [Fact] + public async Task DoesNotRetrySuccessfulPull() + { + var attempts = 0; + var retries = 0; + var delays = 0; + + await DockerImagePullRetryPolicy.ExecuteAsync( + _ => + { + attempts++; + return Task.CompletedTask; + }, + (_, _, _) => retries++, + _ => TimeSpan.Zero, + (_, _) => + { + delays++; + return Task.CompletedTask; + }, + TestContext.Current.CancellationToken) + .ConfigureAwait(true); + + Assert.Equal(1, attempts); + Assert.Equal(0, retries); + Assert.Equal(0, delays); + } + + [Theory] + [MemberData(nameof(TransientExceptions))] + public async Task RetriesTransientFailure(Exception exception) + { + var attempts = 0; + + await DockerImagePullRetryPolicy.ExecuteAsync( + _ => + { + attempts++; + return attempts == 1 ? Task.FromException(exception) : Task.CompletedTask; + }, + (_, _, _) => { }, + _ => TimeSpan.Zero, + (_, _) => Task.CompletedTask, + TestContext.Current.CancellationToken) + .ConfigureAwait(true); + + Assert.Equal(2, attempts); + } + + [Theory] + [MemberData(nameof(PermanentExceptions))] + public async Task DoesNotRetryPermanentFailure(Exception exception) + { + var attempts = 0; + + var actualException = await Assert.ThrowsAsync(exception.GetType(), () => DockerImagePullRetryPolicy.ExecuteAsync( + _ => + { + attempts++; + return Task.FromException(exception); + }, + (_, _, _) => { }, + _ => TimeSpan.Zero, + (_, _) => Task.CompletedTask, + TestContext.Current.CancellationToken)); + + Assert.Same(exception, actualException); + Assert.Equal(1, attempts); + } + + [Fact] + public async Task SucceedsOnFinalAttempt() + { + var attempts = 0; + var retryAttempts = new List(); + var requestedDelays = new List(); + + await DockerImagePullRetryPolicy.ExecuteAsync( + _ => ++attempts < DockerImagePullRetryPolicy.MaxAttempts + ? Task.FromException(new DockerApiException(HttpStatusCode.ServiceUnavailable, null)) + : Task.CompletedTask, + (attempt, delay, _) => + { + retryAttempts.Add(attempt); + requestedDelays.Add(delay); + }, + attempt => TimeSpan.FromSeconds(attempt), + (_, _) => Task.CompletedTask, + TestContext.Current.CancellationToken) + .ConfigureAwait(true); + + Assert.Equal(DockerImagePullRetryPolicy.MaxAttempts, attempts); + Assert.Equal(new[] { 2, 3 }, retryAttempts); + Assert.Equal(new[] { TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2) }, requestedDelays); + } + + [Fact] + public async Task RethrowsFinalFailureAfterMaximumAttempts() + { + var attempts = 0; + var finalException = new DockerApiException(HttpStatusCode.InternalServerError, null); + + var actualException = await Assert.ThrowsAsync(() => DockerImagePullRetryPolicy.ExecuteAsync( + _ => + { + attempts++; + return Task.FromException(finalException); + }, + (_, _, _) => { }, + _ => TimeSpan.Zero, + (_, _) => Task.CompletedTask, + TestContext.Current.CancellationToken)); + + Assert.Same(finalException, actualException); + Assert.Equal(DockerImagePullRetryPolicy.MaxAttempts, attempts); + } + + [Fact] + public async Task StopsWhenCancellationIsRequestedDuringDelay() + { + var attempts = 0; + using var cts = new CancellationTokenSource(); + + await Assert.ThrowsAnyAsync(() => DockerImagePullRetryPolicy.ExecuteAsync( + _ => + { + attempts++; + return Task.FromException(new DockerApiException(HttpStatusCode.InternalServerError, null)); + }, + (_, _, _) => { }, + _ => TimeSpan.Zero, + (_, token) => + { + cts.Cancel(); + return Task.FromCanceled(token); + }, + cts.Token)); + + Assert.Equal(1, attempts); + } + + [Fact] + public async Task DoesNotStartPullWhenAlreadyCanceled() + { + var attempts = 0; + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + await Assert.ThrowsAnyAsync(() => DockerImagePullRetryPolicy.ExecuteAsync( + _ => + { + attempts++; + return Task.CompletedTask; + }, + (_, _, _) => { }, + _ => TimeSpan.Zero, + (_, _) => Task.CompletedTask, + cts.Token)); + + Assert.Equal(0, attempts); + } + + [Fact] + public async Task DoesNotRetryCallerCancellation() + { + var attempts = 0; + using var cts = new CancellationTokenSource(); + + await Assert.ThrowsAnyAsync(() => DockerImagePullRetryPolicy.ExecuteAsync( + _ => + { + attempts++; + cts.Cancel(); + return Task.FromCanceled(cts.Token); + }, + (_, _, _) => { }, + _ => TimeSpan.Zero, + (_, _) => Task.CompletedTask, + cts.Token)); + + Assert.Equal(1, attempts); + } + + [Fact] + public void UsesExponentialBackoffWithBoundedJitter() + { + var firstDelay = DockerImagePullRetryPolicy.GetRetryDelay(1); + var secondDelay = DockerImagePullRetryPolicy.GetRetryDelay(2); + + Assert.InRange(firstDelay, TimeSpan.FromSeconds(1), TimeSpan.FromMilliseconds(1250)); + Assert.InRange(secondDelay, TimeSpan.FromSeconds(2), TimeSpan.FromMilliseconds(2500)); + } + + [Fact] + public async Task ReportsSanitizedDockerApiFailureReason() + { + const string SensitiveResponse = "registry response containing a secret"; + string reason = null; + var attempts = 0; + + await DockerImagePullRetryPolicy.ExecuteAsync( + _ => ++attempts == 1 + ? Task.FromException(new DockerApiException(HttpStatusCode.InternalServerError, SensitiveResponse)) + : Task.CompletedTask, + (_, _, failureReason) => reason = failureReason, + _ => TimeSpan.Zero, + (_, _) => Task.CompletedTask, + TestContext.Current.CancellationToken) + .ConfigureAwait(true); + + Assert.Equal("Docker API status code 500 (InternalServerError)", reason); + Assert.DoesNotContain(SensitiveResponse, reason, StringComparison.Ordinal); + } + } +}