From daad88c53ebe764143c63ad85a9c41f905580b9c Mon Sep 17 00:00:00 2001 From: Hal Spang Date: Fri, 11 Jul 2025 08:43:16 -0700 Subject: [PATCH 1/2] Expose gRPC client retry options Signed-off-by: Hal Spang --- .../DurableTaskSchedulerClientExtensions.cs | 56 ++++++++++++ .../DurableTaskSchedulerClientOptions.cs | 82 ++++++++++++++++- .../Client.AzureManaged.Tests.csproj | 1 + ...rableTaskSchedulerClientExtensionsTests.cs | 87 ++++++++++++++++++- 4 files changed, 223 insertions(+), 3 deletions(-) diff --git a/src/Client/AzureManaged/DurableTaskSchedulerClientExtensions.cs b/src/Client/AzureManaged/DurableTaskSchedulerClientExtensions.cs index b11f450a..995612d8 100644 --- a/src/Client/AzureManaged/DurableTaskSchedulerClientExtensions.cs +++ b/src/Client/AzureManaged/DurableTaskSchedulerClientExtensions.cs @@ -40,15 +40,70 @@ public static void UseDurableTaskScheduler( configure); } + /// + /// Configures Durable Task client to use the Azure Durable Task Scheduler service. + /// + /// The Durable Task client builder to configure. + /// The endpoint address of the Durable Task Scheduler resource. Expected to be in the format "https://{scheduler-name}.{region}.durabletask.io". + /// The name of the task hub resource associated with the Durable Task Scheduler resource. + /// The credential used to authenticate with the Durable Task Scheduler task hub resource. + /// The options that determine how and when a request will be retried. + /// Optional callback to dynamically configure DurableTaskSchedulerClientOptions. + public static void UseDurableTaskScheduler( + this IDurableTaskClientBuilder builder, + string endpointAddress, + string taskHubName, + TokenCredential credential, + DurableTaskSchedulerClientOptions.ClientRetryOptions retryOptions, + Action? configure = null) + { + ConfigureSchedulerOptions( + builder, + options => + { + options.EndpointAddress = endpointAddress; + options.TaskHubName = taskHubName; + options.Credential = credential; + options.RetryOptions = retryOptions; + }, + configure); + } + + /// + /// Configures Durable Task client to use the Azure Durable Task Scheduler service using a connection string. + /// + /// The Durable Task client builder to configure. + /// The connection string used to connect to the Durable Task Scheduler service. + /// Optional callback to dynamically configure DurableTaskSchedulerClientOptions. + public static void UseDurableTaskScheduler( + this IDurableTaskClientBuilder builder, + string connectionString, + Action? configure = null) + { + var connectionOptions = DurableTaskSchedulerClientOptions.FromConnectionString(connectionString); + ConfigureSchedulerOptions( + builder, + options => + { + options.EndpointAddress = connectionOptions.EndpointAddress; + options.TaskHubName = connectionOptions.TaskHubName; + options.Credential = connectionOptions.Credential; + options.AllowInsecureCredentials = connectionOptions.AllowInsecureCredentials; + }, + configure); + } + /// /// Configures Durable Task client to use the Azure Durable Task Scheduler service using a connection string. /// /// The Durable Task client builder to configure. /// The connection string used to connect to the Durable Task Scheduler service. + /// /// The options that determine how and when a request will be retried. /// Optional callback to dynamically configure DurableTaskSchedulerClientOptions. public static void UseDurableTaskScheduler( this IDurableTaskClientBuilder builder, string connectionString, + DurableTaskSchedulerClientOptions.ClientRetryOptions retryOptions, Action? configure = null) { var connectionOptions = DurableTaskSchedulerClientOptions.FromConnectionString(connectionString); @@ -60,6 +115,7 @@ public static void UseDurableTaskScheduler( options.TaskHubName = connectionOptions.TaskHubName; options.Credential = connectionOptions.Credential; options.AllowInsecureCredentials = connectionOptions.AllowInsecureCredentials; + options.RetryOptions = retryOptions; }, configure); } diff --git a/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs b/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs index 360995cc..f3255bda 100644 --- a/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs +++ b/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs @@ -6,8 +6,8 @@ using Azure.Identity; using Grpc.Core; using Grpc.Net.Client; -using Microsoft.DurableTask; using Microsoft.DurableTask.Client; +using grpcConfig = Grpc.Net.Client.Configuration; namespace Microsoft.DurableTask; @@ -46,6 +46,11 @@ public class DurableTaskSchedulerClientOptions /// public bool AllowInsecureCredentials { get; set; } + /// + /// Gets or sets the options that determine how and when calls made to the scheduler will be retried. + /// + internal ClientRetryOptions? RetryOptions { get; set; } + /// /// Creates a new instance of from a connection string. /// @@ -109,6 +114,48 @@ this.Credential is not null metadata.Add("Authorization", $"Bearer {token.Token}"); }); + grpcConfig.ServiceConfig? serviceConfig = null; + if (this.RetryOptions != null) + { + grpcConfig.RetryPolicy retryPolicy = new grpcConfig.RetryPolicy + { + MaxAttempts = this.RetryOptions.MaxRetries ?? GrpcRetryPolicyDefaults.DefaultMaxAttempts, + InitialBackoff = TimeSpan.FromMilliseconds(this.RetryOptions.InitialBackoffMs ?? GrpcRetryPolicyDefaults.DefaultInitialBackoffMs), + MaxBackoff = TimeSpan.FromMilliseconds(this.RetryOptions.MaxBackoffMs ?? GrpcRetryPolicyDefaults.DefaultMaxBackoffMs), + BackoffMultiplier = this.RetryOptions.BackoffMultiplier ?? GrpcRetryPolicyDefaults.DefaultBackoffMultiplier, + RetryableStatusCodes = { StatusCode.Unavailable }, // Always retry on Unavailable. + }; + + if (this.RetryOptions.RetryableStatusCodes != null) + { + foreach (StatusCode statusCode in this.RetryOptions.RetryableStatusCodes) + { + // Added by default, don't need to have it added twice. + if (statusCode == StatusCode.Unavailable) + { + continue; + } + + retryPolicy.RetryableStatusCodes.Add(statusCode); + } + } + + grpcConfig.MethodConfig methodConfig = new grpcConfig.MethodConfig + { + Names = { grpcConfig.MethodName.Default }, + RetryPolicy = retryPolicy, + }; + + serviceConfig = new grpcConfig.ServiceConfig + { + MethodConfigs = { methodConfig }, + }; + } + else + { + serviceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig; + } + // Production will use HTTPS, but local testing will use HTTP ChannelCredentials channelCreds = endpoint.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ? ChannelCredentials.SecureSsl : @@ -117,7 +164,7 @@ this.Credential is not null { Credentials = ChannelCredentials.Create(channelCreds, managedBackendCreds), UnsafeUseInsecureChannelCallCredentials = this.AllowInsecureCredentials, - ServiceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig, + ServiceConfig = serviceConfig, }); } @@ -173,4 +220,35 @@ this.Credential is not null nameof(connectionString)); } } + + /// + /// Options used to configure retries used when making calls to the Scheduler. + /// + public class ClientRetryOptions + { + /// + /// Gets or sets the maximum number of times a call should be retried. + /// + public int? MaxRetries { get; set; } + + /// + /// Gets or sets the initial backoff in milliseconds. + /// + public int? InitialBackoffMs { get; set; } + + /// + /// Gets or sets the maximum backoff in milliseconds. + /// + public int? MaxBackoffMs { get; set; } + + /// + /// Gets or sets the backoff multiplier for exponential backoff. + /// + public double? BackoffMultiplier { get; set; } + + /// + /// Gets or sets the list of status codes that can be retried. + /// + public List? RetryableStatusCodes { get; set; } + } } diff --git a/test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj b/test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj index 22142fa2..4d43922f 100644 --- a/test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj +++ b/test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj @@ -10,6 +10,7 @@ + diff --git a/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs b/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs index d625703c..ec281149 100644 --- a/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs +++ b/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs @@ -4,6 +4,7 @@ using Azure.Core; using Azure.Identity; using FluentAssertions; +using Grpc.Core; using Microsoft.DurableTask.Client.Grpc; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; @@ -109,7 +110,7 @@ public void UseDurableTaskScheduler_WithNullParameters_ShouldThrowOptionsValidat // Assert var action = () => provider.GetRequiredService>().Value; action.Should().Throw() - .WithMessage(endpoint == null + .WithMessage(endpoint == null ? "DataAnnotation validation failed for 'DurableTaskSchedulerClientOptions' members: 'EndpointAddress' with the error: 'Endpoint address is required'." : "DataAnnotation validation failed for 'DurableTaskSchedulerClientOptions' members: 'TaskHubName' with the error: 'Task hub name is required'."); } @@ -193,4 +194,88 @@ public void UseDurableTaskScheduler_WithNamedOptions_ShouldConfigureCorrectly() options.ResourceId.Should().Be("https://durabletask.io"); options.AllowInsecureCredentials.Should().BeFalse(); } + + [Fact] + public void UseDurableTaskScheduler_WithEndpointAndCredentialAndRetryOptions_ShouldConfigureCorrectly() + { + // Arrange + ServiceCollection services = new ServiceCollection(); + Mock mockBuilder = new Mock(); + mockBuilder.Setup(b => b.Services).Returns(services); + DefaultAzureCredential credential = new DefaultAzureCredential(); + + // Act + mockBuilder.Object.UseDurableTaskScheduler(ValidEndpoint, ValidTaskHub, credential, + new DurableTaskSchedulerClientOptions.ClientRetryOptions + { + MaxRetries = 5, + InitialBackoffMs = 100, + MaxBackoffMs = 1000, + BackoffMultiplier = 2.0, + RetryableStatusCodes = new List { StatusCode.Unknown } + }); + + // Assert + ServiceProvider provider = services.BuildServiceProvider(); + IOptions? options = provider.GetService>(); + options.Should().NotBeNull(); + + // Validate the configured options + DurableTaskSchedulerClientOptions clientOptions = provider.GetRequiredService>().Value; + clientOptions.EndpointAddress.Should().Be(ValidEndpoint); + clientOptions.TaskHubName.Should().Be(ValidTaskHub); + clientOptions.Credential.Should().BeOfType(); + clientOptions.RetryOptions.Should().NotBeNull(); + // The assert not null doesn't clear the syntax warning about null checks. + if (clientOptions.RetryOptions != null) + { + clientOptions.RetryOptions.MaxRetries.Should().Be(5); + clientOptions.RetryOptions.InitialBackoffMs.Should().Be(100); + clientOptions.RetryOptions.MaxBackoffMs.Should().Be(1000); + clientOptions.RetryOptions.BackoffMultiplier.Should().Be(2.0); + clientOptions.RetryOptions.RetryableStatusCodes.Should().Contain(StatusCode.Unknown); + } + } + + [Fact] + public void UseDurableTaskScheduler_WithConnectionStringAndRetryOptions_ShouldConfigureCorrectly() + { + // Arrange + ServiceCollection services = new ServiceCollection(); + Mock mockBuilder = new Mock(); + mockBuilder.Setup(b => b.Services).Returns(services); + string connectionString = $"Endpoint={ValidEndpoint};Authentication=DefaultAzure;TaskHub={ValidTaskHub}"; + + // Act + mockBuilder.Object.UseDurableTaskScheduler(connectionString, + new DurableTaskSchedulerClientOptions.ClientRetryOptions + { + MaxRetries = 5, + InitialBackoffMs = 100, + MaxBackoffMs = 1000, + BackoffMultiplier = 2.0, + RetryableStatusCodes = new List { StatusCode.Unknown } + }); + + // Assert + ServiceProvider provider = services.BuildServiceProvider(); + IOptions? options = provider.GetService>(); + options.Should().NotBeNull(); + + // Validate the configured options + DurableTaskSchedulerClientOptions clientOptions = provider.GetRequiredService>().Value; + clientOptions.EndpointAddress.Should().Be(ValidEndpoint); + clientOptions.TaskHubName.Should().Be(ValidTaskHub); + clientOptions.Credential.Should().BeOfType(); + clientOptions.RetryOptions.Should().NotBeNull(); + // The assert not null doesn't clear the syntax warning about null checks. + if (clientOptions.RetryOptions != null) + { + clientOptions.RetryOptions.MaxRetries.Should().Be(5); + clientOptions.RetryOptions.InitialBackoffMs.Should().Be(100); + clientOptions.RetryOptions.MaxBackoffMs.Should().Be(1000); + clientOptions.RetryOptions.BackoffMultiplier.Should().Be(2.0); + clientOptions.RetryOptions.RetryableStatusCodes.Should().Contain(StatusCode.Unknown); + } + } } From 668013257eddf1d33f701d5da830186deac912c5 Mon Sep 17 00:00:00 2001 From: Hal Spang Date: Tue, 15 Jul 2025 14:23:31 -0700 Subject: [PATCH 2/2] Remove unneeded constructors, make options public Signed-off-by: Hal Spang --- CHANGELOG.md | 2 + .../DurableTaskSchedulerClientExtensions.cs | 56 ------------------- .../DurableTaskSchedulerClientOptions.cs | 21 +++---- ...rableTaskSchedulerClientExtensionsTests.cs | 38 +++++++------ 4 files changed, 31 insertions(+), 86 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d72ce03..471ae3d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## (Unreleased) +- Expose gRPC retry options in Azure Managed extensions ([#447](https://github.com/microsoft/durabletask-dotnet/pull/447)) + ## v1.12.0 - Activity tag support ([#426](https://github.com/microsoft/durabletask-dotnet/pull/426)) diff --git a/src/Client/AzureManaged/DurableTaskSchedulerClientExtensions.cs b/src/Client/AzureManaged/DurableTaskSchedulerClientExtensions.cs index 995612d8..b11f450a 100644 --- a/src/Client/AzureManaged/DurableTaskSchedulerClientExtensions.cs +++ b/src/Client/AzureManaged/DurableTaskSchedulerClientExtensions.cs @@ -40,70 +40,15 @@ public static void UseDurableTaskScheduler( configure); } - /// - /// Configures Durable Task client to use the Azure Durable Task Scheduler service. - /// - /// The Durable Task client builder to configure. - /// The endpoint address of the Durable Task Scheduler resource. Expected to be in the format "https://{scheduler-name}.{region}.durabletask.io". - /// The name of the task hub resource associated with the Durable Task Scheduler resource. - /// The credential used to authenticate with the Durable Task Scheduler task hub resource. - /// The options that determine how and when a request will be retried. - /// Optional callback to dynamically configure DurableTaskSchedulerClientOptions. - public static void UseDurableTaskScheduler( - this IDurableTaskClientBuilder builder, - string endpointAddress, - string taskHubName, - TokenCredential credential, - DurableTaskSchedulerClientOptions.ClientRetryOptions retryOptions, - Action? configure = null) - { - ConfigureSchedulerOptions( - builder, - options => - { - options.EndpointAddress = endpointAddress; - options.TaskHubName = taskHubName; - options.Credential = credential; - options.RetryOptions = retryOptions; - }, - configure); - } - - /// - /// Configures Durable Task client to use the Azure Durable Task Scheduler service using a connection string. - /// - /// The Durable Task client builder to configure. - /// The connection string used to connect to the Durable Task Scheduler service. - /// Optional callback to dynamically configure DurableTaskSchedulerClientOptions. - public static void UseDurableTaskScheduler( - this IDurableTaskClientBuilder builder, - string connectionString, - Action? configure = null) - { - var connectionOptions = DurableTaskSchedulerClientOptions.FromConnectionString(connectionString); - ConfigureSchedulerOptions( - builder, - options => - { - options.EndpointAddress = connectionOptions.EndpointAddress; - options.TaskHubName = connectionOptions.TaskHubName; - options.Credential = connectionOptions.Credential; - options.AllowInsecureCredentials = connectionOptions.AllowInsecureCredentials; - }, - configure); - } - /// /// Configures Durable Task client to use the Azure Durable Task Scheduler service using a connection string. /// /// The Durable Task client builder to configure. /// The connection string used to connect to the Durable Task Scheduler service. - /// /// The options that determine how and when a request will be retried. /// Optional callback to dynamically configure DurableTaskSchedulerClientOptions. public static void UseDurableTaskScheduler( this IDurableTaskClientBuilder builder, string connectionString, - DurableTaskSchedulerClientOptions.ClientRetryOptions retryOptions, Action? configure = null) { var connectionOptions = DurableTaskSchedulerClientOptions.FromConnectionString(connectionString); @@ -115,7 +60,6 @@ public static void UseDurableTaskScheduler( options.TaskHubName = connectionOptions.TaskHubName; options.Credential = connectionOptions.Credential; options.AllowInsecureCredentials = connectionOptions.AllowInsecureCredentials; - options.RetryOptions = retryOptions; }, configure); } diff --git a/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs b/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs index f3255bda..3a00ba7a 100644 --- a/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs +++ b/src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs @@ -7,7 +7,7 @@ using Grpc.Core; using Grpc.Net.Client; using Microsoft.DurableTask.Client; -using grpcConfig = Grpc.Net.Client.Configuration; +using GrpcConfig = Grpc.Net.Client.Configuration; namespace Microsoft.DurableTask; @@ -49,7 +49,7 @@ public class DurableTaskSchedulerClientOptions /// /// Gets or sets the options that determine how and when calls made to the scheduler will be retried. /// - internal ClientRetryOptions? RetryOptions { get; set; } + public ClientRetryOptions? RetryOptions { get; set; } /// /// Creates a new instance of from a connection string. @@ -114,10 +114,10 @@ this.Credential is not null metadata.Add("Authorization", $"Bearer {token.Token}"); }); - grpcConfig.ServiceConfig? serviceConfig = null; + GrpcConfig.ServiceConfig? serviceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig; if (this.RetryOptions != null) { - grpcConfig.RetryPolicy retryPolicy = new grpcConfig.RetryPolicy + GrpcConfig.RetryPolicy retryPolicy = new GrpcConfig.RetryPolicy { MaxAttempts = this.RetryOptions.MaxRetries ?? GrpcRetryPolicyDefaults.DefaultMaxAttempts, InitialBackoff = TimeSpan.FromMilliseconds(this.RetryOptions.InitialBackoffMs ?? GrpcRetryPolicyDefaults.DefaultInitialBackoffMs), @@ -140,21 +140,18 @@ this.Credential is not null } } - grpcConfig.MethodConfig methodConfig = new grpcConfig.MethodConfig + GrpcConfig.MethodConfig methodConfig = new GrpcConfig.MethodConfig { - Names = { grpcConfig.MethodName.Default }, + // MethodName.Default applies this retry policy configuration to all gRPC methods on the channel. + Names = { GrpcConfig.MethodName.Default }, RetryPolicy = retryPolicy, }; - serviceConfig = new grpcConfig.ServiceConfig + serviceConfig = new GrpcConfig.ServiceConfig { MethodConfigs = { methodConfig }, }; } - else - { - serviceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig; - } // Production will use HTTPS, but local testing will use HTTP ChannelCredentials channelCreds = endpoint.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ? @@ -249,6 +246,6 @@ public class ClientRetryOptions /// /// Gets or sets the list of status codes that can be retried. /// - public List? RetryableStatusCodes { get; set; } + public IList? RetryableStatusCodes { get; set; } } } diff --git a/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs b/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs index ec281149..7aad801c 100644 --- a/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs +++ b/test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs @@ -205,15 +205,16 @@ public void UseDurableTaskScheduler_WithEndpointAndCredentialAndRetryOptions_Sho DefaultAzureCredential credential = new DefaultAzureCredential(); // Act - mockBuilder.Object.UseDurableTaskScheduler(ValidEndpoint, ValidTaskHub, credential, - new DurableTaskSchedulerClientOptions.ClientRetryOptions - { - MaxRetries = 5, - InitialBackoffMs = 100, - MaxBackoffMs = 1000, - BackoffMultiplier = 2.0, - RetryableStatusCodes = new List { StatusCode.Unknown } - }); + mockBuilder.Object.UseDurableTaskScheduler(ValidEndpoint, ValidTaskHub, credential, options => + options.RetryOptions = new DurableTaskSchedulerClientOptions.ClientRetryOptions + { + MaxRetries = 5, + InitialBackoffMs = 100, + MaxBackoffMs = 1000, + BackoffMultiplier = 2.0, + RetryableStatusCodes = new List { StatusCode.Unknown } + } + ); // Assert ServiceProvider provider = services.BuildServiceProvider(); @@ -247,15 +248,16 @@ public void UseDurableTaskScheduler_WithConnectionStringAndRetryOptions_ShouldCo string connectionString = $"Endpoint={ValidEndpoint};Authentication=DefaultAzure;TaskHub={ValidTaskHub}"; // Act - mockBuilder.Object.UseDurableTaskScheduler(connectionString, - new DurableTaskSchedulerClientOptions.ClientRetryOptions - { - MaxRetries = 5, - InitialBackoffMs = 100, - MaxBackoffMs = 1000, - BackoffMultiplier = 2.0, - RetryableStatusCodes = new List { StatusCode.Unknown } - }); + mockBuilder.Object.UseDurableTaskScheduler(connectionString, options => + options.RetryOptions = new DurableTaskSchedulerClientOptions.ClientRetryOptions + { + MaxRetries = 5, + InitialBackoffMs = 100, + MaxBackoffMs = 1000, + BackoffMultiplier = 2.0, + RetryableStatusCodes = new List { StatusCode.Unknown } + } + ); // Assert ServiceProvider provider = services.BuildServiceProvider();