fix: Retry transient Docker image pull failures

src/Testcontainers/Clients/DockerImageOperations.cs

@@ -70,7 +70,10 @@ public async Task CreateAsync(IImage image, IDockerRegistryAuthenticationConfigu
         IdentityToken = dockerRegistryAuthConfig.IdentityToken,
       };
 
-      await DockerClient.Images.CreateImageAsync(createParameters, authConfig, traceProgress, ct)
+      await DockerImagePullRetryPolicy.ExecuteAsync(
+          token => DockerClient.Images.CreateImageAsync(createParameters, authConfig, traceProgress, token),
+          (attempt, delay, reason) => Logger.RetryDockerImagePull(image, attempt, DockerImagePullRetryPolicy.MaxAttempts, delay, reason),
+          ct)
         .ConfigureAwait(false);
 
       Logger.DockerImageCreated(image);

src/Testcontainers/Clients/DockerImagePullRetryPolicy.cs

@@ -0,0 +1,93 @@
+namespace DotNet.Testcontainers.Clients
+{
+  using System;
+  using System.IO;
+  using System.Net;
+  using System.Net.Http;
+  using System.Net.Sockets;
+  using System.Threading;
+  using System.Threading.Tasks;
+  using Docker.DotNet;
+
+  internal static class DockerImagePullRetryPolicy
+  {
+    internal const int MaxAttempts = 3;
+
+    private const int InitialDelayInMilliseconds = 1000;
+
+    private static readonly Random Random = new Random();
+
+    public static Task ExecuteAsync(Func<CancellationToken, Task> pull, Action<int, TimeSpan, string> onRetry, CancellationToken ct)
+    {
+      return ExecuteAsync(pull, onRetry, GetRetryDelay, (delay, token) => Task.Delay(delay, token), ct);
+    }
+
+    internal static async Task ExecuteAsync(Func<CancellationToken, Task> pull, Action<int, TimeSpan, string> onRetry, Func<int, TimeSpan> getRetryDelay, Func<TimeSpan, CancellationToken, Task> delay, CancellationToken ct)
+    {
+      for (var attempt = 1; ; attempt++)
+      {
+        ct.ThrowIfCancellationRequested();
+
+        try
+        {
+          await pull(ct)
+            .ConfigureAwait(false);
+
+          return;
+        }
+        catch (Exception exception) when (attempt < MaxAttempts && IsTransient(exception, ct))
+        {
+          var retryDelay = getRetryDelay(attempt);
+          onRetry(attempt + 1, retryDelay, GetFailureReason(exception));
+
+          await delay(retryDelay, ct)
+            .ConfigureAwait(false);
+        }
+      }
+    }
+
+    internal static bool IsTransient(Exception exception, CancellationToken ct)
+    {
+      if (exception is DockerApiException dockerApiException)
+      {
+        var statusCode = (int)dockerApiException.StatusCode;
+        return HttpStatusCode.RequestTimeout.Equals(dockerApiException.StatusCode)
+          || (HttpStatusCode)429 == dockerApiException.StatusCode
+          || statusCode >= 500 && statusCode <= 599;
+      }
+
+      if (exception is OperationCanceledException)
+      {
+        return !ct.IsCancellationRequested;
+      }
+
+      return exception is HttpRequestException
+        || exception is IOException
+        || exception is SocketException
+        || exception is TimeoutException;
+    }
+
+    internal static TimeSpan GetRetryDelay(int attempt)
+    {
+      var exponentialDelay = InitialDelayInMilliseconds * (1 << (attempt - 1));
+      int jitter;
+
+      lock (Random)
+      {
+        jitter = Random.Next(0, exponentialDelay / 4 + 1);
+      }
+
+      return TimeSpan.FromMilliseconds(exponentialDelay + jitter);
+    }
+
+    private static string GetFailureReason(Exception exception)
+    {
+      if (exception is DockerApiException dockerApiException)
+      {
+        return $"Docker API status code {(int)dockerApiException.StatusCode} ({dockerApiException.StatusCode})";
+      }
+
+      return exception.GetType().Name;
+    }
+  }
+}

src/Testcontainers/Logging.cs

@@ -69,6 +69,9 @@ internal static partial class Logging
     [LoggerMessage(Level = LogLevel.Information, Message = "Docker image {FullName} created")]
     private static partial void DockerImageCreatedCore(ILogger logger, string fullName);
 
+    [LoggerMessage(Level = LogLevel.Warning, Message = "Retry Docker image {FullName} pull, attempt {Attempt}/{MaxAttempts} in {Delay} due to {Reason}")]
+    private static partial void RetryDockerImagePullCore(ILogger logger, string fullName, int attempt, int maxAttempts, TimeSpan delay, string reason);
+
     [LoggerMessage(Level = LogLevel.Information, Message = "Docker image {FullName} built")]
     private static partial void DockerImageBuiltCore(ILogger logger, string fullName);
 
@@ -214,6 +217,11 @@ public static void DockerImageCreated(this ILogger logger, IImage image)
       DockerImageCreatedCore(logger, image.FullName);
     }
 
+    public static void RetryDockerImagePull(this ILogger logger, IImage image, int attempt, int maxAttempts, TimeSpan delay, string reason)
+    {
+      RetryDockerImagePullCore(logger, image.FullName, attempt, maxAttempts, delay, reason);
+    }
+
     public static void DockerImageBuilt(this ILogger logger, IImage image)
     {
       DockerImageBuiltCore(logger, image.FullName);

tests/Testcontainers.Tests/Unit/Clients/DockerImagePullRetryPolicyTest.cs

@@ -0,0 +1,261 @@
+namespace DotNet.Testcontainers.Tests.Unit
+{
+  using System;
+  using System.Collections.Generic;
+  using System.IO;
+  using System.Linq;
+  using System.Net;
+  using System.Net.Http;
+  using System.Net.Sockets;
+  using System.Threading;
+  using System.Threading.Tasks;
+  using Docker.DotNet;
+  using DotNet.Testcontainers.Clients;
+  using Xunit;
+
+  public sealed class DockerImagePullRetryPolicyTest
+  {
+    public static IEnumerable<object[]> TransientExceptions { get; } = new Exception[]
+    {
+      new DockerApiException(HttpStatusCode.RequestTimeout, null),
+      new DockerApiException((HttpStatusCode)429, null),
+      new DockerApiException(HttpStatusCode.InternalServerError, null),
+      new DockerApiException(HttpStatusCode.NotImplemented, null),
+      new DockerApiException(HttpStatusCode.BadGateway, null),
+      new DockerApiException(HttpStatusCode.ServiceUnavailable, null),
+      new DockerApiException(HttpStatusCode.GatewayTimeout, null),
+      new DockerApiException((HttpStatusCode)599, null),
+      new HttpRequestException(),
+      new IOException(),
+      new SocketException(),
+      new TimeoutException(),
+      new TaskCanceledException(),
+    }.Select(exception => new object[] { exception });
+
+    public static IEnumerable<object[]> PermanentExceptions { get; } = new Exception[]
+    {
+      new DockerApiException(HttpStatusCode.BadRequest, null),
+      new DockerApiException(HttpStatusCode.Unauthorized, null),
+      new DockerApiException(HttpStatusCode.Forbidden, null),
+      new DockerApiException(HttpStatusCode.NotFound, null),
+      new DockerApiException(HttpStatusCode.Conflict, null),
+      new DockerApiException((HttpStatusCode)422, null),
+      new DockerApiException((HttpStatusCode)600, null),
+      new InvalidOperationException(),
+    }.Select(exception => new object[] { exception });
+
+    [Fact]
+    public async Task DoesNotRetrySuccessfulPull()
+    {
+      var attempts = 0;
+      var retries = 0;
+      var delays = 0;
+
+      await DockerImagePullRetryPolicy.ExecuteAsync(
+          _ =>
+          {
+            attempts++;
+            return Task.CompletedTask;
+          },
+          (_, _, _) => retries++,
+          _ => TimeSpan.Zero,
+          (_, _) =>
+          {
+            delays++;
+            return Task.CompletedTask;
+          },
+          TestContext.Current.CancellationToken)
+        .ConfigureAwait(true);
+
+      Assert.Equal(1, attempts);
+      Assert.Equal(0, retries);
+      Assert.Equal(0, delays);
+    }
+
+    [Theory]
+    [MemberData(nameof(TransientExceptions))]
+    public async Task RetriesTransientFailure(Exception exception)
+    {
+      var attempts = 0;
+
+      await DockerImagePullRetryPolicy.ExecuteAsync(
+          _ =>
+          {
+            attempts++;
+            return attempts == 1 ? Task.FromException(exception) : Task.CompletedTask;
+          },
+          (_, _, _) => { },
+          _ => TimeSpan.Zero,
+          (_, _) => Task.CompletedTask,
+          TestContext.Current.CancellationToken)
+        .ConfigureAwait(true);
+
+      Assert.Equal(2, attempts);
+    }
+
+    [Theory]
+    [MemberData(nameof(PermanentExceptions))]
+    public async Task DoesNotRetryPermanentFailure(Exception exception)
+    {
+      var attempts = 0;
+
+      var actualException = await Assert.ThrowsAsync(exception.GetType(), () => DockerImagePullRetryPolicy.ExecuteAsync(
+          _ =>
+          {
+            attempts++;
+            return Task.FromException(exception);
+          },
+          (_, _, _) => { },
+          _ => TimeSpan.Zero,
+          (_, _) => Task.CompletedTask,
+          TestContext.Current.CancellationToken));
+
+      Assert.Same(exception, actualException);
+      Assert.Equal(1, attempts);
+    }
+
+    [Fact]
+    public async Task SucceedsOnFinalAttempt()
+    {
+      var attempts = 0;
+      var retryAttempts = new List<int>();
+      var requestedDelays = new List<TimeSpan>();
+
+      await DockerImagePullRetryPolicy.ExecuteAsync(
+          _ => ++attempts < DockerImagePullRetryPolicy.MaxAttempts
+            ? Task.FromException(new DockerApiException(HttpStatusCode.ServiceUnavailable, null))
+            : Task.CompletedTask,
+          (attempt, delay, _) =>
+          {
+            retryAttempts.Add(attempt);
+            requestedDelays.Add(delay);
+          },
+          attempt => TimeSpan.FromSeconds(attempt),
+          (_, _) => Task.CompletedTask,
+          TestContext.Current.CancellationToken)
+        .ConfigureAwait(true);
+
+      Assert.Equal(DockerImagePullRetryPolicy.MaxAttempts, attempts);
+      Assert.Equal(new[] { 2, 3 }, retryAttempts);
+      Assert.Equal(new[] { TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(2) }, requestedDelays);
+    }
+
+    [Fact]
+    public async Task RethrowsFinalFailureAfterMaximumAttempts()
+    {
+      var attempts = 0;
+      var finalException = new DockerApiException(HttpStatusCode.InternalServerError, null);
+
+      var actualException = await Assert.ThrowsAsync<DockerApiException>(() => DockerImagePullRetryPolicy.ExecuteAsync(
+          _ =>
+          {
+            attempts++;
+            return Task.FromException(finalException);
+          },
+          (_, _, _) => { },
+          _ => TimeSpan.Zero,
+          (_, _) => Task.CompletedTask,
+          TestContext.Current.CancellationToken));
+
+      Assert.Same(finalException, actualException);
+      Assert.Equal(DockerImagePullRetryPolicy.MaxAttempts, attempts);
+    }
+
+    [Fact]
+    public async Task StopsWhenCancellationIsRequestedDuringDelay()
+    {
+      var attempts = 0;
+      using var cts = new CancellationTokenSource();
+
+      await Assert.ThrowsAnyAsync<OperationCanceledException>(() => DockerImagePullRetryPolicy.ExecuteAsync(
+          _ =>
+          {
+            attempts++;
+            return Task.FromException(new DockerApiException(HttpStatusCode.InternalServerError, null));
+          },
+          (_, _, _) => { },
+          _ => TimeSpan.Zero,
+          (_, token) =>
+          {
+            cts.Cancel();
+            return Task.FromCanceled(token);
+          },
+          cts.Token));
+
+      Assert.Equal(1, attempts);
+    }
+
+    [Fact]
+    public async Task DoesNotStartPullWhenAlreadyCanceled()
+    {
+      var attempts = 0;
+      using var cts = new CancellationTokenSource();
+      cts.Cancel();
+
+      await Assert.ThrowsAnyAsync<OperationCanceledException>(() => DockerImagePullRetryPolicy.ExecuteAsync(
+          _ =>
+          {
+            attempts++;
+            return Task.CompletedTask;
+          },
+          (_, _, _) => { },
+          _ => TimeSpan.Zero,
+          (_, _) => Task.CompletedTask,
+          cts.Token));
+
+      Assert.Equal(0, attempts);
+    }
+
+    [Fact]
+    public async Task DoesNotRetryCallerCancellation()
+    {
+      var attempts = 0;
+      using var cts = new CancellationTokenSource();
+
+      await Assert.ThrowsAnyAsync<OperationCanceledException>(() => DockerImagePullRetryPolicy.ExecuteAsync(
+          _ =>
+          {
+            attempts++;
+            cts.Cancel();
+            return Task.FromCanceled(cts.Token);
+          },
+          (_, _, _) => { },
+          _ => TimeSpan.Zero,
+          (_, _) => Task.CompletedTask,
+          cts.Token));
+
+      Assert.Equal(1, attempts);
+    }
+
+    [Fact]
+    public void UsesExponentialBackoffWithBoundedJitter()
+    {
+      var firstDelay = DockerImagePullRetryPolicy.GetRetryDelay(1);
+      var secondDelay = DockerImagePullRetryPolicy.GetRetryDelay(2);
+
+      Assert.InRange(firstDelay, TimeSpan.FromSeconds(1), TimeSpan.FromMilliseconds(1250));
+      Assert.InRange(secondDelay, TimeSpan.FromSeconds(2), TimeSpan.FromMilliseconds(2500));
+    }
+
+    [Fact]
+    public async Task ReportsSanitizedDockerApiFailureReason()
+    {
+      const string SensitiveResponse = "registry response containing a secret";
+      string reason = null;
+      var attempts = 0;
+
+      await DockerImagePullRetryPolicy.ExecuteAsync(
+          _ => ++attempts == 1
+            ? Task.FromException(new DockerApiException(HttpStatusCode.InternalServerError, SensitiveResponse))
+            : Task.CompletedTask,
+          (_, _, failureReason) => reason = failureReason,
+          _ => TimeSpan.Zero,
+          (_, _) => Task.CompletedTask,
+          TestContext.Current.CancellationToken)
+        .ConfigureAwait(true);
+
+      Assert.Equal("Docker API status code 500 (InternalServerError)", reason);
+      Assert.DoesNotContain(SensitiveResponse, reason, StringComparison.Ordinal);
+    }
+  }
+}