diff --git a/Microsoft.DurableTask.sln b/Microsoft.DurableTask.sln index 383f2e15..53f5f4bc 100644 --- a/Microsoft.DurableTask.sln +++ b/Microsoft.DurableTask.sln @@ -97,6 +97,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LargePayloadConsoleApp", "s EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureBlobPayloads", "src\Extensions\AzureBlobPayloads\AzureBlobPayloads.csproj", "{FE1DA748-D6DB-E168-BC42-6DBBCEAF229C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "InProcessTestHost", "src\InProcessTestHost\InProcessTestHost.csproj", "{5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "InProcessTestHost.Tests", "test\InProcessTestHost.Tests\InProcessTestHost.Tests.csproj", "{B894780C-338F-475E-8E84-56AFA8197A06}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -251,6 +255,14 @@ Global {D2779F32-A548-44F8-B60A-6AC018966C79}.Debug|Any CPU.Build.0 = Debug|Any CPU {D2779F32-A548-44F8-B60A-6AC018966C79}.Release|Any CPU.ActiveCfg = Release|Any CPU {D2779F32-A548-44F8-B60A-6AC018966C79}.Release|Any CPU.Build.0 = Release|Any CPU + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E}.Release|Any CPU.Build.0 = Release|Any CPU + {B894780C-338F-475E-8E84-56AFA8197A06}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B894780C-338F-475E-8E84-56AFA8197A06}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B894780C-338F-475E-8E84-56AFA8197A06}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B894780C-338F-475E-8E84-56AFA8197A06}.Release|Any CPU.Build.0 = Release|Any CPU {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}.Debug|Any CPU.Build.0 = Debug|Any CPU {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -305,6 +317,8 @@ Global {A89B766C-987F-4C9F-8937-D0AB9FE640C8} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} {100348B5-4D97-4A3F-B777-AB14F276F8FE} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} {D2779F32-A548-44F8-B60A-6AC018966C79} = {E5637F81-2FB9-4CD7-900D-455363B142A7} + {5F1E1662-D2D1-4325-BFE3-6AE23A8A4D7E} = {8AFC9781-F6F1-4696-BB4A-9ED7CA9D612B} + {B894780C-338F-475E-8E84-56AFA8197A06} = {E5637F81-2FB9-4CD7-900D-455363B142A7} {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C} = {8AFC9781-F6F1-4696-BB4A-9ED7CA9D612B} EndGlobalSection diff --git a/src/InProcessTestHost/DurableTaskTestHost.cs b/src/InProcessTestHost/DurableTaskTestHost.cs new file mode 100644 index 00000000..ec075b94 --- /dev/null +++ b/src/InProcessTestHost/DurableTaskTestHost.cs @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using DurableTask.Core; +using Grpc.Net.Client; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Testing.Sidecar; +using Microsoft.DurableTask.Testing.Sidecar.Grpc; +using Microsoft.DurableTask.Worker; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.Testing; + +/// +/// In-process test host for testing class-based durable task orchestrations and activities +/// without requiring any external backend (Azure Storage, SQL, etc). +/// +public sealed class DurableTaskTestHost : IAsyncDisposable +{ + readonly IWebHost sidecarHost; + readonly IHost workerHost; + readonly GrpcChannel grpcChannel; + + /// + /// Initializes a new instance of the class. + /// + /// The gRPC sidecar host. + /// The worker host. + /// The gRPC channel. + /// The durable task client. + public DurableTaskTestHost(IWebHost sidecarHost, IHost workerHost, GrpcChannel grpcChannel, DurableTaskClient client) + { + this.sidecarHost = sidecarHost; + this.workerHost = workerHost; + this.grpcChannel = grpcChannel; + this.Client = client; + } + + /// + /// Gets the durable task client for scheduling and managing orchestrations. + /// + public DurableTaskClient Client { get; } + + /// + /// Starts a new in-process test host with the specified orchestrators and activities. + /// + /// Action to configure the task registry by adding orchestrators and activities. + /// Optional configuration options. + /// Cancellation token. + /// A running test host ready to execute orchestrations. + public static async Task StartAsync( + Action registry, + DurableTaskTestHostOptions? options = null, + CancellationToken cancellationToken = default) + { + options ??= new DurableTaskTestHostOptions(); + + // Create in-memory orchestration service + var orchestrationService = new InMemoryOrchestrationService(options.LoggerFactory); + + // Start gRPC sidecar server in-process + string address = options.Port.HasValue + ? $"http://localhost:{options.Port.Value}" + : $"http://localhost:{Random.Shared.Next(30000, 40000)}"; + + var sidecarHost = new WebHostBuilder() + .UseKestrel(kestrelOptions => + { + // Configure for HTTP/2 (required for gRPC) + kestrelOptions.ConfigureEndpointDefaults(listenOptions => + listenOptions.Protocols = HttpProtocols.Http2); + }) + .UseUrls(address) + .ConfigureServices(services => + { + services.AddGrpc(); + services.AddSingleton(orchestrationService); + services.AddSingleton(orchestrationService); + services.AddSingleton(); + }) + .Configure(app => + { + app.UseRouting(); + app.UseEndpoints(endpoints => + { + endpoints.MapGrpcService(); + }); + }) + .Build(); + + sidecarHost.Start(); + var grpcChannel = GrpcChannel.ForAddress(address); + + // Create worker host with user's orchestrators and activities + var workerHost = Host.CreateDefaultBuilder() + .ConfigureLogging(logging => + { + logging.ClearProviders(); + if (options.LoggerFactory != null) + { + logging.Services.AddSingleton(options.LoggerFactory); + } + }) + .ConfigureServices(services => + { + // Register worker that connects to our in-process sidecar + services.AddDurableTaskWorker(builder => + { + builder.UseGrpc(grpcChannel); + builder.AddTasks(registry); + }); + + // Register client that connects to the same sidecar + services.AddDurableTaskClient(builder => + { + builder.UseGrpc(grpcChannel); + builder.RegisterDirectly(); + }); + }) + .Build(); + + await workerHost.StartAsync(cancellationToken); + + // Get the client from the worker host + var client = workerHost.Services.GetRequiredService(); + + return new DurableTaskTestHost(sidecarHost, workerHost, grpcChannel, client); + } + + /// + /// Clean up all resources. + /// + /// A task representing the asynchronous dispose operation. + public async ValueTask DisposeAsync() + { + await this.workerHost.StopAsync(); + this.workerHost.Dispose(); + + await this.grpcChannel.ShutdownAsync(); + this.grpcChannel.Dispose(); + + await this.sidecarHost.StopAsync(); + this.sidecarHost.Dispose(); + } +} + +/// +/// Configuration options for . +/// +public class DurableTaskTestHostOptions +{ + /// + /// Gets or sets the specific port to use for the gRPC sidecar. + /// If not set, a random port between 30000-40000 will be used. + /// + public int? Port { get; set; } + + /// + /// Gets or sets an optional logger factory for capturing logs during tests. + /// Null by default. + /// + public ILoggerFactory? LoggerFactory { get; set; } +} diff --git a/src/InProcessTestHost/InProcessTestHost.csproj b/src/InProcessTestHost/InProcessTestHost.csproj new file mode 100644 index 00000000..8839ef0f --- /dev/null +++ b/src/InProcessTestHost/InProcessTestHost.csproj @@ -0,0 +1,27 @@ + + + + net6.0 + Microsoft.DurableTask.Testing + Microsoft.DurableTask.InProcessTestHost + Microsoft.DurableTask.InProcessTestHost + 0.1.0-preview.1 + + + $(NoWarn);CA1848 + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/InProcessTestHost/README.md b/src/InProcessTestHost/README.md new file mode 100644 index 00000000..754f556a --- /dev/null +++ b/src/InProcessTestHost/README.md @@ -0,0 +1,43 @@ +# DurableTaskTestHost - Testing Durable Orchestrations In-Process + +`DurableTaskTestHost` is a simple API for testing your durable task orchestrations and activities **in-process** without requiring any external backend. + +Supports both **class-based** and **function-based** syntax. + +## Quick Start + +1. Configure options +```csharp +var options = new DurableTaskTestHostOptions +{ + Port = 31000, // Optional: specific port (random by default) + LoggerFactory = myLoggerFactory // Optional: pass logger factory for logging +}; + +``` + +2. Register test orchestrations and activities. + +```csharp +await using var testHost = await DurableTaskTestHost.StartAsync(registry => +{ + // Class-based + registry.AddOrchestrator(); + registry.AddActivity(); + + // Function-based + registry.AddOrchestratorFunc("MyFunc", (ctx, input) => Task.FromResult("done")); + registry.AddActivityFunc("MyActivity", (ctx, input) => Task.FromResult("result")); +}); + +``` + +3. Test +```csharp +string instanceId = await testHost.Client.ScheduleNewOrchestrationInstanceAsync("MyOrchestrator"); +var result = await testHost.Client.WaitForInstanceCompletionAsync(instanceId); +``` + . +## More Samples + +See [BasicOrchestrationTests.cs](../../test/InProcessTestHost.Tests/BasicOrchestrationTests.cs) for complete samples showing both class-syntax and function-syntax orchestrations. diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/AsyncManualResetEvent.cs b/src/InProcessTestHost/Sidecar/AsyncManualResetEvent.cs similarity index 62% rename from test/Grpc.IntegrationTests/GrpcSidecar/AsyncManualResetEvent.cs rename to src/InProcessTestHost/Sidecar/AsyncManualResetEvent.cs index b7cf6dc4..4589a584 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/AsyncManualResetEvent.cs +++ b/src/InProcessTestHost/Sidecar/AsyncManualResetEvent.cs @@ -1,13 +1,20 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -namespace Microsoft.DurableTask.Sidecar; +namespace Microsoft.DurableTask.Testing.Sidecar; +/// +/// Helper class for fetching TaskHub events. +/// class AsyncManualResetEvent { readonly object mutex = new(); TaskCompletionSource tcs = new(TaskCreationOptions.RunContinuationsAsynchronously); + /// + /// Initializes a new instance of the class. + /// + /// Whether the event should start in the signaled state. public AsyncManualResetEvent(bool isSignaled) { if (isSignaled) @@ -16,6 +23,17 @@ public AsyncManualResetEvent(bool isSignaled) } } + /// + /// Gets a value indicating whether the event is in the signaled state. + /// + public bool IsSignaled => this.tcs.Task.IsCompleted; + + /// + /// Waits for the event to be signaled with a timeout. + /// + /// The timeout duration. + /// The cancellation token. + /// True if the event was signaled, false if the timeout occurred. public async Task WaitAsync(TimeSpan timeout, CancellationToken cancellationToken) { Task delayTask = Task.Delay(timeout, cancellationToken); @@ -29,11 +47,10 @@ public async Task WaitAsync(TimeSpan timeout, CancellationToken cancellati return winner == waitTask; } - public bool IsSignaled => this.tcs.Task.IsCompleted; - /// /// Puts the event in the signaled state, unblocking any waiting threads. /// + /// True if result is set. public bool Set() { lock (this.mutex) diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcCreateSubOrchestrationAction.cs b/src/InProcessTestHost/Sidecar/Dispatcher/GrpcCreateSubOrchestrationAction.cs similarity index 56% rename from test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcCreateSubOrchestrationAction.cs rename to src/InProcessTestHost/Sidecar/Dispatcher/GrpcCreateSubOrchestrationAction.cs index ce916d7a..9c22e132 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcCreateSubOrchestrationAction.cs +++ b/src/InProcessTestHost/Sidecar/Dispatcher/GrpcCreateSubOrchestrationAction.cs @@ -4,9 +4,15 @@ using DurableTask.Core.Command; using DurableTask.Core.Tracing; -namespace Microsoft.DurableTask.Sidecar.Dispatcher; +namespace Microsoft.DurableTask.Testing.Sidecar.Dispatcher; +/// +/// Action for creating sub-orchestration. +/// public class GrpcCreateSubOrchestrationAction : CreateSubOrchestrationAction { + /// + /// Gets or sets distributed parent trace context. + /// public DistributedTraceContext? ParentTraceContext { get; set; } } diff --git a/src/InProcessTestHost/Sidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs b/src/InProcessTestHost/Sidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs new file mode 100644 index 00000000..77ff454d --- /dev/null +++ b/src/InProcessTestHost/Sidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using DurableTask.Core; + +namespace Microsoft.DurableTask.Testing.Sidecar.Dispatcher; + +/// +/// Grpc orchestration execution result. +/// +public class GrpcOrchestratorExecutionResult : OrchestratorExecutionResult +{ + /// + /// Gets or sets the orcehstration activity spanId. + /// + public string? OrchestrationActivitySpanId { get; set; } + + /// + /// Gets or sets the orchestration activity start time. + /// + public DateTimeOffset? OrchestrationActivityStartTime { get; set; } +} diff --git a/src/InProcessTestHost/Sidecar/Dispatcher/GrpcScheduleTaskOrchestratorAction.cs b/src/InProcessTestHost/Sidecar/Dispatcher/GrpcScheduleTaskOrchestratorAction.cs new file mode 100644 index 00000000..c7646f93 --- /dev/null +++ b/src/InProcessTestHost/Sidecar/Dispatcher/GrpcScheduleTaskOrchestratorAction.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using DurableTask.Core.Command; +using DurableTask.Core.Tracing; + +namespace Microsoft.DurableTask.Testing.Sidecar.Dispatcher; + +/// +/// gRPC-specific implementation of ScheduleTaskOrchestratorAction that includes distributed tracing context. +/// +public class GrpcScheduleTaskOrchestratorAction : ScheduleTaskOrchestratorAction +{ + /// + /// Gets or sets the parent trace context for distributed tracing. + /// + public DistributedTraceContext? ParentTraceContext { get; set; } +} diff --git a/src/InProcessTestHost/Sidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs b/src/InProcessTestHost/Sidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs new file mode 100644 index 00000000..e502b369 --- /dev/null +++ b/src/InProcessTestHost/Sidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Runtime.Serialization; +using DurableTask.Core.History; +using DurableTask.Core.Tracing; + +namespace Microsoft.DurableTask.Testing.Sidecar.Dispatcher; + +/// +/// gRPC-specific implementation of SubOrchestrationInstanceCreatedEvent that includes distributed tracing context. +/// +public class GrpcSubOrchestrationInstanceCreatedEvent : SubOrchestrationInstanceCreatedEvent +{ + /// + /// Initializes a new instance of the class. + /// + /// The event ID. + public GrpcSubOrchestrationInstanceCreatedEvent(int eventId) + : base(eventId) + { + } + + /// + /// Gets or sets the parent trace context for distributed tracing. + /// + [DataMember] + public DistributedTraceContext? ParentTraceContext { get; set; } +} diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/ITaskExecutor.cs b/src/InProcessTestHost/Sidecar/Dispatcher/ITaskExecutor.cs similarity index 91% rename from test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/ITaskExecutor.cs rename to src/InProcessTestHost/Sidecar/Dispatcher/ITaskExecutor.cs index 2a8f144e..cfb2f76d 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/ITaskExecutor.cs +++ b/src/InProcessTestHost/Sidecar/Dispatcher/ITaskExecutor.cs @@ -1,11 +1,14 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using DurableTask.Core; using DurableTask.Core.History; -namespace Microsoft.DurableTask.Sidecar.Dispatcher; +namespace Microsoft.DurableTask.Testing.Sidecar.Dispatcher; +/// +/// Task Executor. +/// interface ITaskExecutor { /// diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/ITrafficSignal.cs b/src/InProcessTestHost/Sidecar/Dispatcher/ITrafficSignal.cs similarity index 78% rename from test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/ITrafficSignal.cs rename to src/InProcessTestHost/Sidecar/Dispatcher/ITrafficSignal.cs index fa6a092b..b073fe65 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/ITrafficSignal.cs +++ b/src/InProcessTestHost/Sidecar/Dispatcher/ITrafficSignal.cs @@ -1,7 +1,7 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -namespace Microsoft.DurableTask.Sidecar.Dispatcher; +namespace Microsoft.DurableTask.Testing.Sidecar.Dispatcher; /// /// A simple primitive that can be used to block logical threads until some condition occurs. @@ -9,12 +9,12 @@ namespace Microsoft.DurableTask.Sidecar.Dispatcher; interface ITrafficSignal { /// - /// Provides a human-friendly reason for why the signal is in the "wait" state. + /// Gets a human-friendly reason for why the signal is in the "wait" state. /// string WaitReason { get; } /// - /// Blocks the caller until the method is called. + /// Blocks the caller until the Set method is called. /// /// The amount of time to wait until the signal is unblocked. /// A cancellation token that can be used to interrupt a waiting caller. @@ -26,4 +26,3 @@ interface ITrafficSignal /// Task WaitAsync(TimeSpan waitTime, CancellationToken cancellationToken); } - diff --git a/src/InProcessTestHost/Sidecar/Dispatcher/TaskActivityDispatcher.cs b/src/InProcessTestHost/Sidecar/Dispatcher/TaskActivityDispatcher.cs new file mode 100644 index 00000000..bb728255 --- /dev/null +++ b/src/InProcessTestHost/Sidecar/Dispatcher/TaskActivityDispatcher.cs @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using DurableTask.Core; +using DurableTask.Core.History; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.Testing.Sidecar.Dispatcher; + +/// +/// Dispatches and manages the execution of instances. +/// +class TaskActivityDispatcher : WorkItemDispatcher +{ + readonly IOrchestrationService service; + readonly ITaskExecutor taskExecutor; + + /// + /// Initializes a new instance of the class. + /// + /// The logger used for diagnostic output. + /// A signal used to control dispatcher activity. + /// The orchestration service that manages task activity work items. + /// The task executor responsible for running activity code. + public TaskActivityDispatcher(ILogger log, ITrafficSignal trafficSignal, IOrchestrationService service, ITaskExecutor taskExecutor) + : base(log, trafficSignal) + { + this.service = service; + this.taskExecutor = taskExecutor; + } + + /// + /// Gets the maximum number of concurrent work items allowed by the underlying orchestration service. + /// + public override int MaxWorkItems => this.service.MaxConcurrentTaskActivityWorkItems; + + /// + /// Abandons a task activity work item, releasing any associated locks. + /// + /// The work item to abandon. + /// A task that represents the asynchronous operation. + public override Task AbandonWorkItemAsync(TaskActivityWorkItem workItem) => + this.service.AbandonTaskActivityWorkItemAsync(workItem); + + /// + /// Attempts to fetch the next available from the orchestration service. + /// + /// The maximum duration to wait for an available work item. + /// A token to signal operation cancellation. + /// + /// A task that returns the locked work item, or null if none is available. + /// + public override Task FetchWorkItemAsync(TimeSpan timeout, CancellationToken cancellationToken) => + this.service.LockNextTaskActivityWorkItem(timeout, cancellationToken); + + /// + /// Determines the delay, in seconds, to wait before retrying after a fetch exception occurs. + /// + /// The exception that occurred during fetch. + /// The delay duration in seconds before the next fetch attempt. + public override int GetDelayInSecondsOnFetchException(Exception ex) => + this.service.GetDelayInSecondsAfterOnFetchException(ex); + + /// + /// Retrieves a unique identifier for the specified work item. + /// + /// The work item for which to retrieve the ID. + /// The unique identifier for the work item. + public override string GetWorkItemId(TaskActivityWorkItem workItem) => workItem.Id; + + /// + /// Not Implemented. + /// + /// The work item to release. + /// A completed task. + public override Task ReleaseWorkItemAsync(TaskActivityWorkItem workItem) => Task.CompletedTask; + + /// + /// Renews the lock on the specified task activity work item to prevent expiration. + /// + /// The work item to renew. + /// + /// A task that returns the renewed work item upon successful lock renewal. + /// + public override Task RenewWorkItemAsync(TaskActivityWorkItem workItem) => + this.service.RenewTaskActivityWorkItemLockAsync(workItem); + + /// + /// Executes the specified using the configured . + /// + /// The work item to execute. + /// A task that represents the asynchronous operation. + protected override async Task ExecuteWorkItemAsync(TaskActivityWorkItem workItem) + { + TaskScheduledEvent scheduledEvent = (TaskScheduledEvent)workItem.TaskMessage.Event; + + // TODO: Error handling for internal errors (user code exceptions are handled by the executor). + ActivityExecutionResult result = await this.taskExecutor.ExecuteActivity( + instance: workItem.TaskMessage.OrchestrationInstance, + activityEvent: scheduledEvent); + + TaskMessage responseMessage = new() + { + Event = result.ResponseEvent, + OrchestrationInstance = workItem.TaskMessage.OrchestrationInstance, + }; + + await this.service.CompleteTaskActivityWorkItemAsync(workItem, responseMessage); + } +} diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskHubDispatcherHost.cs b/src/InProcessTestHost/Sidecar/Dispatcher/TaskHubDispatcherHost.cs similarity index 76% rename from test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskHubDispatcherHost.cs rename to src/InProcessTestHost/Sidecar/Dispatcher/TaskHubDispatcherHost.cs index 89a54d02..6946fc0f 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskHubDispatcherHost.cs +++ b/src/InProcessTestHost/Sidecar/Dispatcher/TaskHubDispatcherHost.cs @@ -1,12 +1,12 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using DurableTask.Core; using Microsoft.Extensions.Logging; -namespace Microsoft.DurableTask.Sidecar.Dispatcher; +namespace Microsoft.DurableTask.Testing.Sidecar.Dispatcher; -class TaskHubDispatcherHost +class TaskHubDispatcherHost : IDisposable { readonly TaskOrchestrationDispatcher orchestrationDispatcher; readonly TaskActivityDispatcher activityDispatcher; @@ -23,8 +23,8 @@ public TaskHubDispatcherHost( this.orchestrationService = orchestrationService ?? throw new ArgumentNullException(nameof(orchestrationService)); this.log = loggerFactory.CreateLogger("Microsoft.DurableTask.Sidecar"); - this.orchestrationDispatcher = new TaskOrchestrationDispatcher(log, trafficSignal, orchestrationService, taskExecutor); - this.activityDispatcher = new TaskActivityDispatcher(log, trafficSignal, orchestrationService, taskExecutor); + this.orchestrationDispatcher = new TaskOrchestrationDispatcher(this.log, trafficSignal, orchestrationService, taskExecutor); + this.activityDispatcher = new TaskActivityDispatcher(this.log, trafficSignal, orchestrationService, taskExecutor); } public async Task StartAsync(CancellationToken cancellationToken) @@ -48,4 +48,12 @@ await Task.WhenAll( // Tell the storage provider to stop doing any background work. await this.orchestrationService.StopAsync(); } + + public void Dispose() + { + // Dispose owned disposable resources + this.activityDispatcher?.Dispose(); + this.orchestrationDispatcher?.Dispose(); + } } + diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs b/src/InProcessTestHost/Sidecar/Dispatcher/TaskOrchestrationDispatcher.cs similarity index 81% rename from test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs rename to src/InProcessTestHost/Sidecar/Dispatcher/TaskOrchestrationDispatcher.cs index 3b16d1c4..f19ca532 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskOrchestrationDispatcher.cs +++ b/src/InProcessTestHost/Sidecar/Dispatcher/TaskOrchestrationDispatcher.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using System.Text; @@ -7,14 +7,24 @@ using DurableTask.Core.History; using Microsoft.Extensions.Logging; -namespace Microsoft.DurableTask.Sidecar.Dispatcher; +namespace Microsoft.DurableTask.Testing.Sidecar.Dispatcher; +/// +/// Dispatches and manages the execution of instances. +/// class TaskOrchestrationDispatcher : WorkItemDispatcher { readonly ILogger log; readonly IOrchestrationService service; readonly ITaskExecutor taskExecutor; + /// + /// Initializes a new instance of the class. + /// + /// The logger used for diagnostic output. + /// A signal used to control dispatcher activity and flow. + /// The orchestration service used to fetch and manage orchestration work items. + /// The task executor used for executing orchestration tasks. public TaskOrchestrationDispatcher(ILogger log, ITrafficSignal trafficSignal, IOrchestrationService service, ITaskExecutor taskExecutor) : base(log, trafficSignal) { @@ -23,14 +33,70 @@ public TaskOrchestrationDispatcher(ILogger log, ITrafficSignal trafficSignal, IO this.taskExecutor = taskExecutor; } + /// + /// Gets the maximum number of concurrent orchestration work items allowed by the orchestration service. + /// public override int MaxWorkItems => this.service.MaxConcurrentTaskOrchestrationWorkItems; + /// + /// Abandons the specified orchestration work item, releasing any held locks or resources. + /// + /// The orchestration work item to abandon. + /// A task that represents the asynchronous abandon operation. public override Task AbandonWorkItemAsync(TaskOrchestrationWorkItem workItem) => this.service.AbandonTaskOrchestrationWorkItemAsync(workItem); + /// + /// Attempts to fetch the next available from the orchestration service. + /// + /// The maximum duration to wait for an available orchestration work item. + /// A token to signal cancellation of the fetch operation. + /// + /// A task that returns the locked orchestration work item if available, or null if no items are ready. + /// public override Task FetchWorkItemAsync(TimeSpan timeout, CancellationToken cancellationToken) => this.service.LockNextTaskOrchestrationWorkItemAsync(timeout, cancellationToken); + /// + /// Determines the delay, in seconds, before retrying after a fetch exception. + /// + /// The exception that occurred while fetching work. + /// The number of seconds to delay before retrying. + public override int GetDelayInSecondsOnFetchException(Exception ex) => + this.service.GetDelayInSecondsAfterOnFetchException(ex); + + /// + /// Get work item id. + /// + /// The orchestration work item. + /// Work item id. + public override string GetWorkItemId(TaskOrchestrationWorkItem workItem) => workItem.InstanceId; + + /// + /// Releases the specified orchestration work item. + /// + /// The work item to release. + /// A task that completes when the release is finished. + public override Task ReleaseWorkItemAsync(TaskOrchestrationWorkItem workItem) => + this.service.ReleaseTaskOrchestrationWorkItemAsync(workItem); + + /// + /// Renew work item. + /// + /// The work item to be renewed. + /// Work item renewed. + public override async Task RenewWorkItemAsync(TaskOrchestrationWorkItem workItem) + { + await this.service.RenewTaskOrchestrationWorkItemLockAsync(workItem); + return workItem; + } + + /// + /// Execute the work item. + /// + /// Work item to execute. + /// Completed task. + /// Throw when work item doesn't contain required message. protected override async Task ExecuteWorkItemAsync(TaskOrchestrationWorkItem workItem) { // Convert the new messages into new history events @@ -72,9 +138,9 @@ protected override async Task ExecuteWorkItemAsync(TaskOrchestrationWorkItem wor this.ApplyOrchestratorActions( result, ref workItem.OrchestrationRuntimeState, - out IList activityMessages, - out IList orchestratorMessages, - out IList timerMessages, + out List activityMessages, + out List orchestratorMessages, + out List timerMessages, out OrchestrationState? updatedStatus, out bool continueAsNew); if (continueAsNew) @@ -166,12 +232,24 @@ static int GetSortOrderWithinGroup(TaskMessage msg) } } + static string GetShortHistoryEventDescription(HistoryEvent e) + { + if (Utils.TryGetTaskScheduledId(e, out int taskScheduledId)) + { + return $"{e.EventType}#{taskScheduledId}"; + } + else + { + return e.EventType.ToString(); + } + } + void ApplyOrchestratorActions( OrchestratorExecutionResult result, ref OrchestrationRuntimeState runtimeState, - out IList activityMessages, - out IList orchestratorMessages, - out IList timerMessages, + out List activityMessages, // CA1859: Use concrete types for better performance + out List orchestratorMessages, // CA1859: Use concrete types for better performance + out List timerMessages, // CA1859: Use concrete types for better performance out OrchestrationState? updatedStatus, out bool continueAsNew) { @@ -180,9 +258,9 @@ void ApplyOrchestratorActions( throw new ArgumentException($"The provided {nameof(OrchestrationRuntimeState)} doesn't contain an instance ID!", nameof(runtimeState)); } - IList? newActivityMessages = null; - IList? newTimerMessages = null; - IList? newOrchestratorMessages = null; + List? newActivityMessages = null; // CA1859: Use concrete types for better performance + List? newTimerMessages = null; // CA1859: Use concrete types for better performance + List? newOrchestratorMessages = null; // CA1859: Use concrete types for better performance FailureDetails? failureDetails = null; continueAsNew = false; @@ -245,7 +323,7 @@ void ApplyOrchestratorActions( Version = subOrchestrationAction.Version, InstanceId = subOrchestrationAction.InstanceId, Input = subOrchestrationAction.Input, - ParentTraceContext = grpcAction?.ParentTraceContext + ParentTraceContext = grpcAction?.ParentTraceContext, }); ExecutionStartedEvent startedEvent = new(-1, subOrchestrationAction.Input) @@ -315,7 +393,7 @@ void ApplyOrchestratorActions( Tags = runtimeState.Tags, ParentInstance = runtimeState.ParentInstance, Name = runtimeState.Name, - Version = completeAction.NewVersion ?? runtimeState.Version + Version = completeAction.NewVersion ?? runtimeState.Version, }); newRuntimeState.Status = runtimeState.Status; @@ -330,9 +408,9 @@ void ApplyOrchestratorActions( } runtimeState = newRuntimeState; - activityMessages = Array.Empty(); - orchestratorMessages = Array.Empty(); - timerMessages = Array.Empty(); + activityMessages = new List(); + orchestratorMessages = new List(); + timerMessages = new List(); continueAsNew = true; updatedStatus = null; return; @@ -397,9 +475,9 @@ void ApplyOrchestratorActions( runtimeState.AddEvent(new OrchestratorCompletedEvent(-1)); - activityMessages = newActivityMessages ?? Array.Empty(); - timerMessages = newTimerMessages ?? Array.Empty(); - orchestratorMessages = newOrchestratorMessages ?? Array.Empty(); + activityMessages = newActivityMessages ?? new List(); + timerMessages = newTimerMessages ?? new List(); + orchestratorMessages = newOrchestratorMessages ?? new List(); updatedStatus = new OrchestrationState { @@ -421,30 +499,4 @@ void ApplyOrchestratorActions( FailureDetails = failureDetails, }; } - - static string GetShortHistoryEventDescription(HistoryEvent e) - { - if (Utils.TryGetTaskScheduledId(e, out int taskScheduledId)) - { - return $"{e.EventType}#{taskScheduledId}"; - } - else - { - return e.EventType.ToString(); - } - } - - public override int GetDelayInSecondsOnFetchException(Exception ex) => - this.service.GetDelayInSecondsAfterOnFetchException(ex); - - public override string GetWorkItemId(TaskOrchestrationWorkItem workItem) => workItem.InstanceId; - - public override Task ReleaseWorkItemAsync(TaskOrchestrationWorkItem workItem) => - this.service.ReleaseTaskOrchestrationWorkItemAsync(workItem); - - public override async Task RenewWorkItemAsync(TaskOrchestrationWorkItem workItem) - { - await this.service.RenewTaskOrchestrationWorkItemLockAsync(workItem); - return workItem; - } } diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/WorkItemDispatcher.cs b/src/InProcessTestHost/Sidecar/Dispatcher/WorkItemDispatcher.cs similarity index 71% rename from test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/WorkItemDispatcher.cs rename to src/InProcessTestHost/Sidecar/Dispatcher/WorkItemDispatcher.cs index 3a9039bb..d026bc3b 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/WorkItemDispatcher.cs +++ b/src/InProcessTestHost/Sidecar/Dispatcher/WorkItemDispatcher.cs @@ -1,14 +1,18 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using System.Diagnostics; using Microsoft.Extensions.Logging; -namespace Microsoft.DurableTask.Sidecar.Dispatcher; +namespace Microsoft.DurableTask.Testing.Sidecar.Dispatcher; -abstract class WorkItemDispatcher where T : class +/// +/// Base class for dispatching and managing work items. +/// +/// The type of work item to dispatch. +abstract class WorkItemDispatcher : IDisposable where T : class { - static int nextDispatcherId = 0; + static int nextDispatcherId; // CA1805: Remove explicit initialization readonly string name; readonly ILogger log; @@ -17,7 +21,13 @@ abstract class WorkItemDispatcher where T : class CancellationTokenSource? shutdownTcs; Task? workItemExecuteLoop; int currentWorkItems; + bool disposed; + /// + /// Initializes a new instance of the class. + /// + /// The logger. + /// The traffic signal. public WorkItemDispatcher(ILogger log, ITrafficSignal trafficSignal) { this.log = log ?? throw new ArgumentNullException(nameof(log)); @@ -26,22 +36,66 @@ public WorkItemDispatcher(ILogger log, ITrafficSignal trafficSignal) this.name = $"{this.GetType().Name}-{Interlocked.Increment(ref nextDispatcherId)}"; } + /// + /// Gets the maximum number of concurrent work items. + /// public virtual int MaxWorkItems => 10; + /// + /// Fetches the next work item asynchronously. + /// + /// The timeout for the operation. + /// The cancellation token. + /// The next work item, or null if none available. public abstract Task FetchWorkItemAsync(TimeSpan timeout, CancellationToken cancellationToken); + /// + /// Executes the specified work item. + /// + /// The work item to execute. + /// A task representing the execution. protected abstract Task ExecuteWorkItemAsync(T workItem); + /// + /// Releases the specified work item. + /// + /// The work item to release. + /// A task representing the release operation. public abstract Task ReleaseWorkItemAsync(T workItem); + /// + /// Abandons the specified work item. + /// + /// The work item to abandon. + /// A task representing the abandon operation. public abstract Task AbandonWorkItemAsync(T workItem); + /// + /// Renews the specified work item. + /// + /// The work item to renew. + /// The renewed work item. public abstract Task RenewWorkItemAsync(T workItem); + /// + /// Gets the ID of the specified work item. + /// + /// The work item. + /// The work item ID. public abstract string GetWorkItemId(T workItem); + /// + /// Gets the delay in seconds after a fetch exception. + /// + /// The exception that occurred. + /// The delay in seconds. public abstract int GetDelayInSecondsOnFetchException(Exception ex); + /// + /// Starts the dispatcher. + /// + /// The cancellation token. + /// A task representing the start operation. public virtual Task StartAsync(CancellationToken cancellationToken) { // Dispatchers can be stopped and started back up again @@ -55,6 +109,11 @@ public virtual Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } + /// + /// Stops the dispatcher. + /// + /// The cancellation token. + /// A task representing the stop operation. public virtual async Task StopAsync(CancellationToken cancellationToken) { // Trigger the cancellation tokens being used for background processing. @@ -71,6 +130,28 @@ public virtual async Task StopAsync(CancellationToken cancellationToken) await this.WaitForOutstandingWorkItems(cancellationToken); } + /// + /// Disposes the dispatcher resources. + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Disposes the dispatcher resources. + /// + /// Whether disposing from Dispose method. + protected virtual void Dispose(bool disposing) + { + if (!this.disposed && disposing) + { + this.shutdownTcs?.Dispose(); + this.disposed = true; + } + } + async Task WaitForAllClear(CancellationToken cancellationToken) { TimeSpan logInterval = TimeSpan.FromMinutes(1); @@ -128,11 +209,12 @@ async Task WaitForOutstandingWorkItems(CancellationToken cancellationToken) } // This method does not throw + // CA1068: CancellationToken should be last parameter async Task DelayOnException( Exception exception, string workItemId, - CancellationToken cancellationToken, - Func delayInSecondsPolicy) + Func delayInSecondsPolicy, + CancellationToken cancellationToken) { try { @@ -208,7 +290,7 @@ async Task FetchAndExecuteLoop(CancellationToken cancellationToken) action: "fetchWorkItem", workItemId: unknownWorkItemId, details: ex.ToString()); - await this.DelayOnException(ex, unknownWorkItemId, cancellationToken, this.GetDelayInSecondsOnFetchException); + await this.DelayOnException(ex, unknownWorkItemId, this.GetDelayInSecondsOnFetchException, cancellationToken); continue; } } diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs b/src/InProcessTestHost/Sidecar/Grpc/ProtobufUtils.cs similarity index 88% rename from test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs rename to src/InProcessTestHost/Sidecar/Grpc/ProtobufUtils.cs index 86b98030..fe4e093c 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/ProtobufUtils.cs +++ b/src/InProcessTestHost/Sidecar/Grpc/ProtobufUtils.cs @@ -1,547 +1,598 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Buffers; -using System.Collections; -using System.Globalization; -using System.Text.Json; -using DurableTask.Core; -using DurableTask.Core.Command; -using DurableTask.Core.History; -using DurableTask.Core.Query; -using DurableTask.Core.Tracing; -using Google.Protobuf; -using Google.Protobuf.Collections; -using Google.Protobuf.WellKnownTypes; -using Microsoft.DurableTask.Sidecar.Dispatcher; -using Proto = Microsoft.DurableTask.Protobuf; - -namespace Microsoft.DurableTask.Sidecar.Grpc; - -public static class ProtobufUtils -{ - public static Proto.HistoryEvent ToHistoryEventProto(HistoryEvent e) - { - var payload = new Proto.HistoryEvent() - { - EventId = e.EventId, - Timestamp = Timestamp.FromDateTime(e.Timestamp), - }; - - switch (e.EventType) - { - case EventType.ContinueAsNew: - var continueAsNew = (ContinueAsNewEvent)e; - payload.ContinueAsNew = new Proto.ContinueAsNewEvent - { - Input = continueAsNew.Result, - }; - break; - case EventType.EventRaised: - var eventRaised = (EventRaisedEvent)e; - payload.EventRaised = new Proto.EventRaisedEvent - { - Name = eventRaised.Name, - Input = eventRaised.Input, - }; - break; - case EventType.EventSent: - var eventSent = (EventSentEvent)e; - payload.EventSent = new Proto.EventSentEvent - { - Name = eventSent.Name, - Input = eventSent.Input, - InstanceId = eventSent.InstanceId, - }; - break; - case EventType.ExecutionCompleted: - var completedEvent = (ExecutionCompletedEvent)e; - payload.ExecutionCompleted = new Proto.ExecutionCompletedEvent - { - OrchestrationStatus = Proto.OrchestrationStatus.Completed, - Result = completedEvent.Result, - }; - break; - case EventType.ExecutionFailed: - var failedEvent = (ExecutionCompletedEvent)e; - payload.ExecutionCompleted = new Proto.ExecutionCompletedEvent - { - OrchestrationStatus = Proto.OrchestrationStatus.Failed, - Result = failedEvent.Result, - }; - break; - case EventType.ExecutionStarted: - // Start of a new orchestration instance - var startedEvent = (ExecutionStartedEvent)e; - startedEvent.Tags ??= new Dictionary(); - payload.ExecutionStarted = new Proto.ExecutionStartedEvent - { - Name = startedEvent.Name, - Version = startedEvent.Version, - Input = startedEvent.Input, - Tags = { startedEvent.Tags }, - OrchestrationInstance = new Proto.OrchestrationInstance - { - InstanceId = startedEvent.OrchestrationInstance.InstanceId, - ExecutionId = startedEvent.OrchestrationInstance.ExecutionId, - }, - ParentInstance = startedEvent.ParentInstance == null ? null : new Proto.ParentInstanceInfo - { - Name = startedEvent.ParentInstance.Name, - Version = startedEvent.ParentInstance.Version, - TaskScheduledId = startedEvent.ParentInstance.TaskScheduleId, - OrchestrationInstance = new Proto.OrchestrationInstance - { - InstanceId = startedEvent.ParentInstance.OrchestrationInstance.InstanceId, - ExecutionId = startedEvent.ParentInstance.OrchestrationInstance.ExecutionId, - }, - }, - ScheduledStartTimestamp = startedEvent.ScheduledStartTime == null ? null : Timestamp.FromDateTime(startedEvent.ScheduledStartTime.Value), - ParentTraceContext = startedEvent.ParentTraceContext is null ? null : new Proto.TraceContext - { - TraceParent = startedEvent.ParentTraceContext.TraceParent, - TraceState = startedEvent.ParentTraceContext.TraceState, - } - }; - break; - case EventType.ExecutionTerminated: - var terminatedEvent = (ExecutionTerminatedEvent)e; - payload.ExecutionTerminated = new Proto.ExecutionTerminatedEvent - { - Input = terminatedEvent.Input, - }; - break; - case EventType.TaskScheduled: - var taskScheduledEvent = (TaskScheduledEvent)e; - payload.TaskScheduled = new Proto.TaskScheduledEvent - { - Name = taskScheduledEvent.Name, - Version = taskScheduledEvent.Version, - Input = taskScheduledEvent.Input, - ParentTraceContext = taskScheduledEvent.ParentTraceContext is null ? null : new Proto.TraceContext - { - TraceParent = taskScheduledEvent.ParentTraceContext.TraceParent, - TraceState = taskScheduledEvent.ParentTraceContext.TraceState, - }, - }; - break; - case EventType.TaskCompleted: - var taskCompletedEvent = (TaskCompletedEvent)e; - payload.TaskCompleted = new Proto.TaskCompletedEvent - { - Result = taskCompletedEvent.Result, - TaskScheduledId = taskCompletedEvent.TaskScheduledId, - }; - break; - case EventType.TaskFailed: - var taskFailedEvent = (TaskFailedEvent)e; - payload.TaskFailed = new Proto.TaskFailedEvent - { - FailureDetails = GetFailureDetails(taskFailedEvent.FailureDetails), - TaskScheduledId = taskFailedEvent.TaskScheduledId, - }; - break; - case EventType.SubOrchestrationInstanceCreated: - var subOrchestrationCreated = (SubOrchestrationInstanceCreatedEvent)e; - payload.SubOrchestrationInstanceCreated = new Proto.SubOrchestrationInstanceCreatedEvent - { - Input = subOrchestrationCreated.Input, - InstanceId = subOrchestrationCreated.InstanceId, - Name = subOrchestrationCreated.Name, - Version = subOrchestrationCreated.Version, - }; - - if (subOrchestrationCreated is GrpcSubOrchestrationInstanceCreatedEvent { ParentTraceContext: not null } grpcEvent) - { - payload.SubOrchestrationInstanceCreated.ParentTraceContext = new Proto.TraceContext - { - TraceParent = grpcEvent.ParentTraceContext.TraceParent, - TraceState = grpcEvent.ParentTraceContext.TraceState, - }; - } - - break; - case EventType.SubOrchestrationInstanceCompleted: - var subOrchestrationCompleted = (SubOrchestrationInstanceCompletedEvent)e; - payload.SubOrchestrationInstanceCompleted = new Proto.SubOrchestrationInstanceCompletedEvent - { - Result = subOrchestrationCompleted.Result, - TaskScheduledId = subOrchestrationCompleted.TaskScheduledId, - }; - break; - case EventType.SubOrchestrationInstanceFailed: - var subOrchestrationFailed = (SubOrchestrationInstanceFailedEvent)e; - payload.SubOrchestrationInstanceFailed = new Proto.SubOrchestrationInstanceFailedEvent - { - FailureDetails = GetFailureDetails(subOrchestrationFailed.FailureDetails), - TaskScheduledId = subOrchestrationFailed.TaskScheduledId, - }; - break; - case EventType.TimerCreated: - var timerCreatedEvent = (TimerCreatedEvent)e; - payload.TimerCreated = new Proto.TimerCreatedEvent - { - FireAt = Timestamp.FromDateTime(timerCreatedEvent.FireAt), - }; - break; - case EventType.TimerFired: - var timerFiredEvent = (TimerFiredEvent)e; - payload.TimerFired = new Proto.TimerFiredEvent - { - FireAt = Timestamp.FromDateTime(timerFiredEvent.FireAt), - TimerId = timerFiredEvent.TimerId, - }; - break; - case EventType.OrchestratorStarted: - // This event has no data - payload.OrchestratorStarted = new Proto.OrchestratorStartedEvent(); - break; - case EventType.OrchestratorCompleted: - // This event has no data - payload.OrchestratorCompleted = new Proto.OrchestratorCompletedEvent(); - break; - case EventType.GenericEvent: - var genericEvent = (GenericEvent)e; - payload.GenericEvent = new Proto.GenericEvent - { - Data = genericEvent.Data, - }; - break; - case EventType.HistoryState: - var historyStateEvent = (HistoryStateEvent)e; - payload.HistoryState = new Proto.HistoryStateEvent - { - OrchestrationState = new Proto.OrchestrationState - { - InstanceId = historyStateEvent.State.OrchestrationInstance.InstanceId, - Name = historyStateEvent.State.Name, - Version = historyStateEvent.State.Version, - Input = historyStateEvent.State.Input, - Output = historyStateEvent.State.Output, - ScheduledStartTimestamp = historyStateEvent.State.ScheduledStartTime == null ? null : Timestamp.FromDateTime(historyStateEvent.State.ScheduledStartTime.Value), - CreatedTimestamp = Timestamp.FromDateTime(historyStateEvent.State.CreatedTime), - LastUpdatedTimestamp = Timestamp.FromDateTime(historyStateEvent.State.LastUpdatedTime), - OrchestrationStatus = (Proto.OrchestrationStatus)historyStateEvent.State.OrchestrationStatus, - CustomStatus = historyStateEvent.State.Status, - Tags = { historyStateEvent.State.Tags }, - }, - }; - break; - case EventType.ExecutionSuspended: - var suspendedEvent = (ExecutionSuspendedEvent)e; - payload.ExecutionSuspended = new Proto.ExecutionSuspendedEvent - { - Input = suspendedEvent.Reason, - }; - break; - case EventType.ExecutionResumed: - var resumedEvent = (ExecutionResumedEvent)e; - payload.ExecutionResumed = new Proto.ExecutionResumedEvent - { - Input = resumedEvent.Reason, - }; - break; - default: - throw new NotSupportedException($"Found unsupported history event '{e.EventType}'."); - } - - return payload; - } - - public static OrchestratorAction ToOrchestratorAction(Proto.OrchestratorAction a) - { - switch (a.OrchestratorActionTypeCase) - { - case Proto.OrchestratorAction.OrchestratorActionTypeOneofCase.ScheduleTask: - return new GrpcScheduleTaskOrchestratorAction - { - Id = a.Id, - Input = a.ScheduleTask.Input, - Name = a.ScheduleTask.Name, - Version = a.ScheduleTask.Version, - ParentTraceContext = a.ScheduleTask.ParentTraceContext is not null - ? new DistributedTraceContext(a.ScheduleTask.ParentTraceContext.TraceParent, a.ScheduleTask.ParentTraceContext.TraceState) - : null, - }; - case Proto.OrchestratorAction.OrchestratorActionTypeOneofCase.CreateSubOrchestration: - return new GrpcCreateSubOrchestrationAction - { - Id = a.Id, - Input = a.CreateSubOrchestration.Input, - Name = a.CreateSubOrchestration.Name, - InstanceId = a.CreateSubOrchestration.InstanceId, - ParentTraceContext = a.CreateSubOrchestration.ParentTraceContext is not null - ? new DistributedTraceContext(a.CreateSubOrchestration.ParentTraceContext.TraceParent, a.CreateSubOrchestration.ParentTraceContext.TraceState) - : null, - Tags = null, // TODO - Version = a.CreateSubOrchestration.Version, - }; - case Proto.OrchestratorAction.OrchestratorActionTypeOneofCase.CreateTimer: - return new CreateTimerOrchestratorAction - { - Id = a.Id, - FireAt = a.CreateTimer.FireAt.ToDateTime(), - }; - case Proto.OrchestratorAction.OrchestratorActionTypeOneofCase.SendEvent: - return new SendEventOrchestratorAction - { - Id = a.Id, - Instance = new OrchestrationInstance - { - InstanceId = a.SendEvent.Instance.InstanceId, - ExecutionId = a.SendEvent.Instance.ExecutionId, - }, - EventName = a.SendEvent.Name, - EventData = a.SendEvent.Data, - }; - case Proto.OrchestratorAction.OrchestratorActionTypeOneofCase.CompleteOrchestration: - var completedAction = a.CompleteOrchestration; - var action = new OrchestrationCompleteOrchestratorAction - { - Id = a.Id, - OrchestrationStatus = (OrchestrationStatus)completedAction.OrchestrationStatus, - Result = completedAction.Result, - Details = completedAction.Details, - FailureDetails = GetFailureDetails(completedAction.FailureDetails), - NewVersion = completedAction.NewVersion, - }; - - if (completedAction.CarryoverEvents?.Count > 0) - { - foreach (var e in completedAction.CarryoverEvents) - { - // Only raised events are supported for carryover - if (e.EventRaised is Proto.EventRaisedEvent eventRaised) - { - action.CarryoverEvents.Add(new EventRaisedEvent(e.EventId, eventRaised.Input) - { - Name = eventRaised.Name, - }); - } - - } - } - - return action; - default: - throw new NotSupportedException($"Received unsupported action type '{a.OrchestratorActionTypeCase}'."); - } - } - - public static string Base64Encode(IMessage message) - { - // Create a serialized payload using lower-level protobuf APIs. We do this to avoid allocating - // byte[] arrays for every request, which would otherwise put a heavy burden on the GC. Unfortunately - // the protobuf API version we're using doesn't currently have memory-efficient serialization APIs. - int messageSize = message.CalculateSize(); - byte[] rentedBuffer = ArrayPool.Shared.Rent(messageSize); - try - { - using MemoryStream intermediateBufferStream = new(rentedBuffer, 0, messageSize); - CodedOutputStream protobufOutputStream = new(intermediateBufferStream); - protobufOutputStream.WriteRawMessage(message); - protobufOutputStream.Flush(); - return Convert.ToBase64String(rentedBuffer, 0, messageSize); - } - finally - { - ArrayPool.Shared.Return(rentedBuffer); - } - } - - internal static FailureDetails? GetFailureDetails(Proto.TaskFailureDetails? failureDetails) - { - if (failureDetails == null) - { - return null; - } - - return new FailureDetails( - failureDetails.ErrorType, - failureDetails.ErrorMessage, - failureDetails.StackTrace, - GetFailureDetails(failureDetails.InnerFailure), - failureDetails.IsNonRetriable, - ConvertMapToDictionary(failureDetails.Properties)); - } - - internal static Proto.TaskFailureDetails? GetFailureDetails(FailureDetails? failureDetails) - { - if (failureDetails == null) - { - return null; - } - - var taskFailureDetails = new Proto.TaskFailureDetails - { - ErrorType = failureDetails.ErrorType, - ErrorMessage = failureDetails.ErrorMessage, - StackTrace = failureDetails.StackTrace, - InnerFailure = GetFailureDetails(failureDetails.InnerFailure), - IsNonRetriable = failureDetails.IsNonRetriable, - }; - - // Add properties if they exist - if (failureDetails.Properties != null) - { - foreach (var kvp in failureDetails.Properties) - { - taskFailureDetails.Properties.Add(kvp.Key, ConvertObjectToValue(kvp.Value)); - } - } - - return taskFailureDetails; - } - - internal static OrchestrationQuery ToOrchestrationQuery(Proto.QueryInstancesRequest request) - { - var query = new OrchestrationQuery() - { - RuntimeStatus = request.Query.RuntimeStatus?.Select(status => (OrchestrationStatus)status).ToList(), - CreatedTimeFrom = request.Query.CreatedTimeFrom?.ToDateTime(), - CreatedTimeTo = request.Query.CreatedTimeTo?.ToDateTime(), - TaskHubNames = request.Query.TaskHubNames, - PageSize = request.Query.MaxInstanceCount, - ContinuationToken = request.Query.ContinuationToken, - InstanceIdPrefix = request.Query.InstanceIdPrefix, - FetchInputsAndOutputs = request.Query.FetchInputsAndOutputs, - }; - - return query; - } - - internal static Proto.QueryInstancesResponse CreateQueryInstancesResponse(OrchestrationQueryResult result, Proto.QueryInstancesRequest request) - { - Proto.QueryInstancesResponse response = new Proto.QueryInstancesResponse - { - ContinuationToken = result.ContinuationToken - }; - foreach (OrchestrationState state in result.OrchestrationState) - { - var orchestrationState = new Proto.OrchestrationState - { - InstanceId = state.OrchestrationInstance.InstanceId, - Name = state.Name, - Version = state.Version, - Input = state.Input, - Output = state.Output, - ScheduledStartTimestamp = state.ScheduledStartTime == null ? null : Timestamp.FromDateTime(state.ScheduledStartTime.Value), - CreatedTimestamp = Timestamp.FromDateTime(state.CreatedTime), - LastUpdatedTimestamp = Timestamp.FromDateTime(state.LastUpdatedTime), - OrchestrationStatus = (Proto.OrchestrationStatus)state.OrchestrationStatus, - CustomStatus = state.Status, - }; - response.OrchestrationState.Add(orchestrationState); - } - return response; - } - - internal static PurgeInstanceFilter ToPurgeInstanceFilter(Proto.PurgeInstancesRequest request) - { - var purgeInstanceFilter = new PurgeInstanceFilter( - request.PurgeInstanceFilter.CreatedTimeFrom.ToDateTime(), - request.PurgeInstanceFilter.CreatedTimeTo?.ToDateTime(), - request.PurgeInstanceFilter.RuntimeStatus?.Select(status => (OrchestrationStatus)status).ToList() - ); - return purgeInstanceFilter; - } - - internal static Proto.PurgeInstancesResponse CreatePurgeInstancesResponse(PurgeResult result) - { - Proto.PurgeInstancesResponse response = new Proto.PurgeInstancesResponse - { - DeletedInstanceCount = result.DeletedInstanceCount - }; - return response; - } - - /// - /// Converts a MapField into a IDictionary. - /// - /// - /// - public static IDictionary ConvertMapToDictionary(MapField properties) - { - return properties.ToDictionary( - kvp => kvp.Key, - kvp => ConvertValueToObject(kvp.Value) - ); - } - - /// - /// Converts a C# object to a protobuf Value. - /// - /// The object to convert. - /// The converted protobuf Value. - internal static Value ConvertObjectToValue(object? obj) - { - return obj switch - { - null => Value.ForNull(), - string str => Value.ForString(str), - bool b => Value.ForBool(b), - int i => Value.ForNumber(i), - long l => Value.ForNumber(l), - float f => Value.ForNumber(f), - double d => Value.ForNumber(d), - decimal dec => Value.ForNumber((double)dec), - - // For DateTime and DateTimeOffset, add prefix to distinguish from normal string. - DateTime dt => Value.ForString($"dt:{dt.ToString("O")}"), - DateTimeOffset dto => Value.ForString($"dto:{dto.ToString("O")}"), - IDictionary dict => Value.ForStruct(new Struct - { - Fields = { dict.ToDictionary(kvp => kvp.Key, kvp => ConvertObjectToValue(kvp.Value)) }, - }), - IEnumerable e => Value.ForList(e.Cast().Select(ConvertObjectToValue).ToArray()), - - // Fallback: convert unlisted type to string. - _ => Value.ForString(obj.ToString() ?? string.Empty), - }; - } - - static object? ConvertValueToObject(Google.Protobuf.WellKnownTypes.Value value) - { - switch (value.KindCase) - { - case Google.Protobuf.WellKnownTypes.Value.KindOneofCase.NullValue: - return null; - case Google.Protobuf.WellKnownTypes.Value.KindOneofCase.NumberValue: - return value.NumberValue; - case Google.Protobuf.WellKnownTypes.Value.KindOneofCase.StringValue: - string stringValue = value.StringValue; - - // If the value starts with the 'dt:' prefix, it may represent a DateTime value — attempt to parse it. - if (stringValue.StartsWith("dt:", StringComparison.Ordinal)) - { - if (DateTime.TryParse(stringValue[3..], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out DateTime date)) - { - return date; - } - } - - // If the value starts with the 'dto:' prefix, it may represent a DateTime value — attempt to parse it. - if (stringValue.StartsWith("dto:", StringComparison.Ordinal)) - { - if (DateTimeOffset.TryParse(stringValue[4..], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out DateTimeOffset date)) - { - return date; - } - } - - // Otherwise just return as string - return stringValue; - case Google.Protobuf.WellKnownTypes.Value.KindOneofCase.BoolValue: - return value.BoolValue; - case Google.Protobuf.WellKnownTypes.Value.KindOneofCase.StructValue: - return value.StructValue.Fields.ToDictionary( - pair => pair.Key, - pair => ConvertValueToObject(pair.Value)); - case Google.Protobuf.WellKnownTypes.Value.KindOneofCase.ListValue: - return value.ListValue.Values.Select(ConvertValueToObject).ToList(); - default: - // Fallback: serialize the whole value to JSON string - return JsonSerializer.Serialize(value); - } - } -} +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Buffers; +using System.Collections; +using System.Globalization; +using System.Text.Json; +using DurableTask.Core; +using DurableTask.Core.Command; +using DurableTask.Core.History; +using DurableTask.Core.Query; +using DurableTask.Core.Tracing; +using Google.Protobuf; +using Google.Protobuf.Collections; +using Google.Protobuf.WellKnownTypes; +using Microsoft.DurableTask.Testing.Sidecar.Dispatcher; +using Proto = Microsoft.DurableTask.Protobuf; + +namespace Microsoft.DurableTask.Testing.Sidecar.Grpc; + +/// +/// Protobuf utils for the in-process grpc service. +/// +public static class ProtobufUtils +{ + /// + /// Convert HistoryEvent to Microsoft.DurableTask.Protobuf.HistoryEvent. + /// + /// The event to convert. + /// Microsoft.DurableTask.Protobuf.HistoryEvent of ths passed event. + /// Throw if the provided event is not supported. + public static Proto.HistoryEvent ToHistoryEventProto(HistoryEvent e) + { + var payload = new Proto.HistoryEvent() + { + EventId = e.EventId, + Timestamp = Timestamp.FromDateTime(e.Timestamp), + }; + + switch (e.EventType) + { + case EventType.ContinueAsNew: + var continueAsNew = (ContinueAsNewEvent)e; + payload.ContinueAsNew = new Proto.ContinueAsNewEvent + { + Input = continueAsNew.Result, + }; + break; + case EventType.EventRaised: + var eventRaised = (EventRaisedEvent)e; + payload.EventRaised = new Proto.EventRaisedEvent + { + Name = eventRaised.Name, + Input = eventRaised.Input, + }; + break; + case EventType.EventSent: + var eventSent = (EventSentEvent)e; + payload.EventSent = new Proto.EventSentEvent + { + Name = eventSent.Name, + Input = eventSent.Input, + InstanceId = eventSent.InstanceId, + }; + break; + case EventType.ExecutionCompleted: + var completedEvent = (ExecutionCompletedEvent)e; + payload.ExecutionCompleted = new Proto.ExecutionCompletedEvent + { + OrchestrationStatus = Proto.OrchestrationStatus.Completed, + Result = completedEvent.Result, + }; + break; + case EventType.ExecutionFailed: + var failedEvent = (ExecutionCompletedEvent)e; + payload.ExecutionCompleted = new Proto.ExecutionCompletedEvent + { + OrchestrationStatus = Proto.OrchestrationStatus.Failed, + Result = failedEvent.Result, + }; + break; + case EventType.ExecutionStarted: + // Start of a new orchestration instance + var startedEvent = (ExecutionStartedEvent)e; + startedEvent.Tags ??= new Dictionary(); + payload.ExecutionStarted = new Proto.ExecutionStartedEvent + { + Name = startedEvent.Name, + Version = startedEvent.Version, + Input = startedEvent.Input, + Tags = { startedEvent.Tags }, + OrchestrationInstance = new Proto.OrchestrationInstance + { + InstanceId = startedEvent.OrchestrationInstance.InstanceId, + ExecutionId = startedEvent.OrchestrationInstance.ExecutionId, + }, + ParentInstance = startedEvent.ParentInstance == null ? null : new Proto.ParentInstanceInfo + { + Name = startedEvent.ParentInstance.Name, + Version = startedEvent.ParentInstance.Version, + TaskScheduledId = startedEvent.ParentInstance.TaskScheduleId, + OrchestrationInstance = new Proto.OrchestrationInstance + { + InstanceId = startedEvent.ParentInstance.OrchestrationInstance.InstanceId, + ExecutionId = startedEvent.ParentInstance.OrchestrationInstance.ExecutionId, + }, + }, + ScheduledStartTimestamp = startedEvent.ScheduledStartTime == null ? null : Timestamp.FromDateTime(startedEvent.ScheduledStartTime.Value), + ParentTraceContext = startedEvent.ParentTraceContext is null ? null : new Proto.TraceContext + { + TraceParent = startedEvent.ParentTraceContext.TraceParent, + TraceState = startedEvent.ParentTraceContext.TraceState, + }, + }; + break; + case EventType.ExecutionTerminated: + var terminatedEvent = (ExecutionTerminatedEvent)e; + payload.ExecutionTerminated = new Proto.ExecutionTerminatedEvent + { + Input = terminatedEvent.Input, + }; + break; + case EventType.TaskScheduled: + var taskScheduledEvent = (TaskScheduledEvent)e; + payload.TaskScheduled = new Proto.TaskScheduledEvent + { + Name = taskScheduledEvent.Name, + Version = taskScheduledEvent.Version, + Input = taskScheduledEvent.Input, + ParentTraceContext = taskScheduledEvent.ParentTraceContext is null ? null : new Proto.TraceContext + { + TraceParent = taskScheduledEvent.ParentTraceContext.TraceParent, + TraceState = taskScheduledEvent.ParentTraceContext.TraceState, + }, + }; + break; + case EventType.TaskCompleted: + var taskCompletedEvent = (TaskCompletedEvent)e; + payload.TaskCompleted = new Proto.TaskCompletedEvent + { + Result = taskCompletedEvent.Result, + TaskScheduledId = taskCompletedEvent.TaskScheduledId, + }; + break; + case EventType.TaskFailed: + var taskFailedEvent = (TaskFailedEvent)e; + payload.TaskFailed = new Proto.TaskFailedEvent + { + FailureDetails = GetFailureDetails(taskFailedEvent.FailureDetails), + TaskScheduledId = taskFailedEvent.TaskScheduledId, + }; + break; + case EventType.SubOrchestrationInstanceCreated: + var subOrchestrationCreated = (SubOrchestrationInstanceCreatedEvent)e; + payload.SubOrchestrationInstanceCreated = new Proto.SubOrchestrationInstanceCreatedEvent + { + Input = subOrchestrationCreated.Input, + InstanceId = subOrchestrationCreated.InstanceId, + Name = subOrchestrationCreated.Name, + Version = subOrchestrationCreated.Version, + }; + + if (subOrchestrationCreated is GrpcSubOrchestrationInstanceCreatedEvent { ParentTraceContext: not null } grpcEvent) + { + payload.SubOrchestrationInstanceCreated.ParentTraceContext = new Proto.TraceContext + { + TraceParent = grpcEvent.ParentTraceContext.TraceParent, + TraceState = grpcEvent.ParentTraceContext.TraceState, + }; + } + + break; + case EventType.SubOrchestrationInstanceCompleted: + var subOrchestrationCompleted = (SubOrchestrationInstanceCompletedEvent)e; + payload.SubOrchestrationInstanceCompleted = new Proto.SubOrchestrationInstanceCompletedEvent + { + Result = subOrchestrationCompleted.Result, + TaskScheduledId = subOrchestrationCompleted.TaskScheduledId, + }; + break; + case EventType.SubOrchestrationInstanceFailed: + var subOrchestrationFailed = (SubOrchestrationInstanceFailedEvent)e; + payload.SubOrchestrationInstanceFailed = new Proto.SubOrchestrationInstanceFailedEvent + { + FailureDetails = GetFailureDetails(subOrchestrationFailed.FailureDetails), + TaskScheduledId = subOrchestrationFailed.TaskScheduledId, + }; + break; + case EventType.TimerCreated: + var timerCreatedEvent = (TimerCreatedEvent)e; + payload.TimerCreated = new Proto.TimerCreatedEvent + { + FireAt = Timestamp.FromDateTime(timerCreatedEvent.FireAt), + }; + break; + case EventType.TimerFired: + var timerFiredEvent = (TimerFiredEvent)e; + payload.TimerFired = new Proto.TimerFiredEvent + { + FireAt = Timestamp.FromDateTime(timerFiredEvent.FireAt), + TimerId = timerFiredEvent.TimerId, + }; + break; + case EventType.OrchestratorStarted: + // This event has no data + payload.OrchestratorStarted = new Proto.OrchestratorStartedEvent(); + break; + case EventType.OrchestratorCompleted: + // This event has no data + payload.OrchestratorCompleted = new Proto.OrchestratorCompletedEvent(); + break; + case EventType.GenericEvent: + var genericEvent = (GenericEvent)e; + payload.GenericEvent = new Proto.GenericEvent + { + Data = genericEvent.Data, + }; + break; + case EventType.HistoryState: + var historyStateEvent = (HistoryStateEvent)e; + payload.HistoryState = new Proto.HistoryStateEvent + { + OrchestrationState = new Proto.OrchestrationState + { + InstanceId = historyStateEvent.State.OrchestrationInstance.InstanceId, + Name = historyStateEvent.State.Name, + Version = historyStateEvent.State.Version, + Input = historyStateEvent.State.Input, + Output = historyStateEvent.State.Output, + ScheduledStartTimestamp = historyStateEvent.State.ScheduledStartTime == null ? null : Timestamp.FromDateTime(historyStateEvent.State.ScheduledStartTime.Value), + CreatedTimestamp = Timestamp.FromDateTime(historyStateEvent.State.CreatedTime), + LastUpdatedTimestamp = Timestamp.FromDateTime(historyStateEvent.State.LastUpdatedTime), + OrchestrationStatus = (Proto.OrchestrationStatus)historyStateEvent.State.OrchestrationStatus, + CustomStatus = historyStateEvent.State.Status, + Tags = { historyStateEvent.State.Tags }, + }, + }; + break; + case EventType.ExecutionSuspended: + var suspendedEvent = (ExecutionSuspendedEvent)e; + payload.ExecutionSuspended = new Proto.ExecutionSuspendedEvent + { + Input = suspendedEvent.Reason, + }; + break; + case EventType.ExecutionResumed: + var resumedEvent = (ExecutionResumedEvent)e; + payload.ExecutionResumed = new Proto.ExecutionResumedEvent + { + Input = resumedEvent.Reason, + }; + break; + default: + throw new NotSupportedException($"Found unsupported history event '{e.EventType}'."); + } + + return payload; + } + + /// + /// Converts an orchestrator action from protobuf format. + /// + /// The protobuf orchestrator action. + /// The converted orchestrator action. + /// Thrown if the action type is not supported. + public static OrchestratorAction ToOrchestratorAction(Proto.OrchestratorAction a) + { + switch (a.OrchestratorActionTypeCase) + { + case Proto.OrchestratorAction.OrchestratorActionTypeOneofCase.ScheduleTask: + return new GrpcScheduleTaskOrchestratorAction + { + Id = a.Id, + Input = a.ScheduleTask.Input, + Name = a.ScheduleTask.Name, + Version = a.ScheduleTask.Version, + ParentTraceContext = a.ScheduleTask.ParentTraceContext is not null + ? new DistributedTraceContext(a.ScheduleTask.ParentTraceContext.TraceParent, a.ScheduleTask.ParentTraceContext.TraceState) + : null, + }; + case Proto.OrchestratorAction.OrchestratorActionTypeOneofCase.CreateSubOrchestration: + return new GrpcCreateSubOrchestrationAction + { + Id = a.Id, + Input = a.CreateSubOrchestration.Input, + Name = a.CreateSubOrchestration.Name, + InstanceId = a.CreateSubOrchestration.InstanceId, + ParentTraceContext = a.CreateSubOrchestration.ParentTraceContext is not null + ? new DistributedTraceContext(a.CreateSubOrchestration.ParentTraceContext.TraceParent, a.CreateSubOrchestration.ParentTraceContext.TraceState) + : null, + Tags = null, // TODO + Version = a.CreateSubOrchestration.Version, + }; + case Proto.OrchestratorAction.OrchestratorActionTypeOneofCase.CreateTimer: + return new CreateTimerOrchestratorAction + { + Id = a.Id, + FireAt = a.CreateTimer.FireAt.ToDateTime(), + }; + case Proto.OrchestratorAction.OrchestratorActionTypeOneofCase.SendEvent: + return new SendEventOrchestratorAction + { + Id = a.Id, + Instance = new OrchestrationInstance + { + InstanceId = a.SendEvent.Instance.InstanceId, + ExecutionId = a.SendEvent.Instance.ExecutionId, + }, + EventName = a.SendEvent.Name, + EventData = a.SendEvent.Data, + }; + case Proto.OrchestratorAction.OrchestratorActionTypeOneofCase.CompleteOrchestration: + var completedAction = a.CompleteOrchestration; + var action = new OrchestrationCompleteOrchestratorAction + { + Id = a.Id, + OrchestrationStatus = (OrchestrationStatus)completedAction.OrchestrationStatus, + Result = completedAction.Result, + Details = completedAction.Details, + FailureDetails = GetFailureDetails(completedAction.FailureDetails), + NewVersion = completedAction.NewVersion, + }; + + if (completedAction.CarryoverEvents?.Count > 0) + { + foreach (var e in completedAction.CarryoverEvents) + { + // Only raised events are supported for carryover + if (e.EventRaised is Proto.EventRaisedEvent eventRaised) + { + action.CarryoverEvents.Add(new EventRaisedEvent(e.EventId, eventRaised.Input) + { + Name = eventRaised.Name, + }); + } + } + } + + return action; + default: + throw new NotSupportedException($"Received unsupported action type '{a.OrchestratorActionTypeCase}'."); + } + } + + /// + /// Base64 encodes a protobuf message. + /// + /// The protobuf message to encode. + /// The base64 encoded string. + public static string Base64Encode(IMessage message) + { + // Create a serialized payload using lower-level protobuf APIs. We do this to avoid allocating + // byte[] arrays for every request, which would otherwise put a heavy burden on the GC. Unfortunately + // the protobuf API version we're using doesn't currently have memory-efficient serialization APIs. + int messageSize = message.CalculateSize(); + byte[] rentedBuffer = ArrayPool.Shared.Rent(messageSize); + try + { + using MemoryStream intermediateBufferStream = new(rentedBuffer, 0, messageSize); + CodedOutputStream protobufOutputStream = new(intermediateBufferStream); + protobufOutputStream.WriteRawMessage(message); + protobufOutputStream.Flush(); + return Convert.ToBase64String(rentedBuffer, 0, messageSize); + } + finally + { + ArrayPool.Shared.Return(rentedBuffer); + } + } + + /// + /// Converts a MapField to a dictionary. + /// + /// The MapField to convert. + /// The converted dictionary. + public static IDictionary ConvertMapToDictionary(MapField properties) + { + return properties.ToDictionary( + kvp => kvp.Key, + kvp => ConvertValueToObject(kvp.Value)); + } + + /// + /// Converts the specified task failure details from proto format to a FailureDetails instance. + /// + /// The task failure details from the proto. + /// + /// A object if is not null; otherwise, null. + /// + internal static FailureDetails? GetFailureDetails(Proto.TaskFailureDetails? failureDetails) + { + if (failureDetails == null) + { + return null; + } + + return new FailureDetails( + failureDetails.ErrorType, + failureDetails.ErrorMessage, + failureDetails.StackTrace, + GetFailureDetails(failureDetails.InnerFailure), + failureDetails.IsNonRetriable, + ConvertMapToDictionary(failureDetails.Properties)); + } + + /// + /// Convert FailureDetails class to proto format. + /// + /// The failure detials to convert. + /// Proto format of failure details. + internal static Proto.TaskFailureDetails? GetFailureDetails(FailureDetails? failureDetails) + { + if (failureDetails == null) + { + return null; + } + + var taskFailureDetails = new Proto.TaskFailureDetails + { + ErrorType = failureDetails.ErrorType, + ErrorMessage = failureDetails.ErrorMessage, + StackTrace = failureDetails.StackTrace, + InnerFailure = GetFailureDetails(failureDetails.InnerFailure), + IsNonRetriable = failureDetails.IsNonRetriable, + }; + + // Add properties if they exist + if (failureDetails.Properties != null) + { + foreach (var kvp in failureDetails.Properties) + { + taskFailureDetails.Properties.Add(kvp.Key, ConvertObjectToValue(kvp.Value)); + } + } + + return taskFailureDetails; + } + + /// + /// Convert QueryInstancesRequest from protobuf format to OrchestrationQuery. + /// + /// Protobuf request to convert. + /// OrchestrationQuery instace. + internal static OrchestrationQuery ToOrchestrationQuery(Proto.QueryInstancesRequest request) + { + var query = new OrchestrationQuery() + { + RuntimeStatus = request.Query.RuntimeStatus?.Select(status => (OrchestrationStatus)status).ToList(), + CreatedTimeFrom = request.Query.CreatedTimeFrom?.ToDateTime(), + CreatedTimeTo = request.Query.CreatedTimeTo?.ToDateTime(), + TaskHubNames = request.Query.TaskHubNames, + PageSize = request.Query.MaxInstanceCount, + ContinuationToken = request.Query.ContinuationToken, + InstanceIdPrefix = request.Query.InstanceIdPrefix, + FetchInputsAndOutputs = request.Query.FetchInputsAndOutputs, + }; + + return query; + } + + /// + /// Creates a protobuf response for an instances query. + /// + /// The query result to serialize. + /// The original request that initiated the query. + /// The populated protobuf response. + internal static Proto.QueryInstancesResponse CreateQueryInstancesResponse(OrchestrationQueryResult result, Proto.QueryInstancesRequest request) + { + Proto.QueryInstancesResponse response = new Proto.QueryInstancesResponse + { + ContinuationToken = result.ContinuationToken, + }; + foreach (OrchestrationState state in result.OrchestrationState) + { + var orchestrationState = new Proto.OrchestrationState + { + InstanceId = state.OrchestrationInstance.InstanceId, + Name = state.Name, + Version = state.Version, + Input = state.Input, + Output = state.Output, + ScheduledStartTimestamp = state.ScheduledStartTime == null ? null : Timestamp.FromDateTime(state.ScheduledStartTime.Value), + CreatedTimestamp = Timestamp.FromDateTime(state.CreatedTime), + LastUpdatedTimestamp = Timestamp.FromDateTime(state.LastUpdatedTime), + OrchestrationStatus = (Proto.OrchestrationStatus)state.OrchestrationStatus, + CustomStatus = state.Status, + }; + response.OrchestrationState.Add(orchestrationState); + } + + return response; + } + + /// + /// Convert PurgeInstancesRequest from protobuf format to PurgeInstanceFilter. + /// + /// Protobuf request to convert. + /// PurgeInstanceFilter instance. + internal static PurgeInstanceFilter ToPurgeInstanceFilter(Proto.PurgeInstancesRequest request) + { + var purgeInstanceFilter = new PurgeInstanceFilter( + request.PurgeInstanceFilter.CreatedTimeFrom.ToDateTime(), + request.PurgeInstanceFilter.CreatedTimeTo?.ToDateTime(), + request.PurgeInstanceFilter.RuntimeStatus?.Select(status => (OrchestrationStatus)status).ToList()); + return purgeInstanceFilter; + } + + /// + /// Creates a protobuf response for a purge operation. + /// + /// The purge result to serialize. + /// The populated protobuf response. + internal static Proto.PurgeInstancesResponse CreatePurgeInstancesResponse(PurgeResult result) + { + Proto.PurgeInstancesResponse response = new Proto.PurgeInstancesResponse + { + DeletedInstanceCount = result.DeletedInstanceCount, + }; + return response; + } + + /// + /// Converts a C# object to a protobuf Value. + /// + /// The object to convert. + /// The converted protobuf Value. + internal static Value ConvertObjectToValue(object? obj) + { + return obj switch + { + null => Value.ForNull(), + string str => Value.ForString(str), + bool b => Value.ForBool(b), + int i => Value.ForNumber(i), + long l => Value.ForNumber(l), + float f => Value.ForNumber(f), + double d => Value.ForNumber(d), + decimal dec => Value.ForNumber((double)dec), + + // For DateTime and DateTimeOffset, add prefix to distinguish from normal string. + DateTime dt => Value.ForString($"dt:{dt.ToString("O")}"), + DateTimeOffset dto => Value.ForString($"dto:{dto.ToString("O")}"), + IDictionary dict => Value.ForStruct(new Struct + { + Fields = { dict.ToDictionary(kvp => kvp.Key, kvp => ConvertObjectToValue(kvp.Value)) }, + }), + IEnumerable e => Value.ForList(e.Cast().Select(ConvertObjectToValue).ToArray()), + + // Fallback: convert unlisted type to string. + _ => Value.ForString(obj.ToString() ?? string.Empty), + }; + } + + static object? ConvertValueToObject(Google.Protobuf.WellKnownTypes.Value value) + { + switch (value.KindCase) + { + case Google.Protobuf.WellKnownTypes.Value.KindOneofCase.NullValue: + return null; + case Google.Protobuf.WellKnownTypes.Value.KindOneofCase.NumberValue: + return value.NumberValue; + case Google.Protobuf.WellKnownTypes.Value.KindOneofCase.StringValue: + string stringValue = value.StringValue; + + // If the value starts with the 'dt:' prefix, it may represent a DateTime value — attempt to parse it. + if (stringValue.StartsWith("dt:", StringComparison.Ordinal)) + { + if (DateTime.TryParse(stringValue[3..], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out DateTime date)) + { + return date; + } + } + + // If the value starts with the 'dto:' prefix, it may represent a DateTime value — attempt to parse it. + if (stringValue.StartsWith("dto:", StringComparison.Ordinal)) + { + if (DateTimeOffset.TryParse(stringValue[4..], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out DateTimeOffset date)) + { + return date; + } + } + + // Otherwise just return as string + return stringValue; + case Google.Protobuf.WellKnownTypes.Value.KindOneofCase.BoolValue: + return value.BoolValue; + case Google.Protobuf.WellKnownTypes.Value.KindOneofCase.StructValue: + return value.StructValue.Fields.ToDictionary( + pair => pair.Key, + pair => ConvertValueToObject(pair.Value)); + case Google.Protobuf.WellKnownTypes.Value.KindOneofCase.ListValue: + return value.ListValue.Values.Select(ConvertValueToObject).ToList(); + default: + // Fallback: serialize the whole value to JSON string + return JsonSerializer.Serialize(value); + } + } +} diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs b/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs similarity index 80% rename from test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs rename to src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs index aba59317..b25177ae 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServer.cs +++ b/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServer.cs @@ -1,760 +1,909 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Collections.Concurrent; -using System.Diagnostics; -using DurableTask.Core; -using DurableTask.Core.History; -using DurableTask.Core.Query; -using Google.Protobuf.WellKnownTypes; -using Grpc.Core; -using Microsoft.DurableTask.Sidecar.Dispatcher; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using P = Microsoft.DurableTask.Protobuf; - -namespace Microsoft.DurableTask.Sidecar.Grpc; - -public class TaskHubGrpcServer : P.TaskHubSidecarService.TaskHubSidecarServiceBase, ITaskExecutor -{ - static readonly Task EmptyCompleteTaskResponse = Task.FromResult(new P.CompleteTaskResponse()); - - readonly ConcurrentDictionary> pendingOrchestratorTasks = new(StringComparer.OrdinalIgnoreCase); - readonly ConcurrentDictionary> pendingActivityTasks = new(StringComparer.OrdinalIgnoreCase); - - readonly ILogger log; - readonly IOrchestrationService service; - readonly IOrchestrationServiceClient client; - readonly IHostApplicationLifetime hostLifetime; - readonly IOptions options; - readonly TaskHubDispatcherHost dispatcherHost; - readonly IsConnectedSignal isConnectedSignal = new(); - readonly SemaphoreSlim sendWorkItemLock = new(initialCount: 1); - readonly ConcurrentDictionary> streamingPastEvents = new(StringComparer.OrdinalIgnoreCase); - - volatile bool supportsHistoryStreaming; - - // Initialized when a client connects to this service to receive work-item commands. - IServerStreamWriter? workerToClientStream; - - public TaskHubGrpcServer( - IHostApplicationLifetime hostApplicationLifetime, - ILoggerFactory loggerFactory, - IOrchestrationService service, - IOrchestrationServiceClient client, - IOptions options) - { - ArgumentNullException.ThrowIfNull(hostApplicationLifetime, nameof(hostApplicationLifetime)); - ArgumentNullException.ThrowIfNull(loggerFactory, nameof(loggerFactory)); - ArgumentNullException.ThrowIfNull(service, nameof(service)); - ArgumentNullException.ThrowIfNull(client, nameof(client)); - ArgumentNullException.ThrowIfNull(options, nameof(options)); - - this.service = service; - this.client = client; - this.log = loggerFactory.CreateLogger("Microsoft.DurableTask.Sidecar"); - this.dispatcherHost = new TaskHubDispatcherHost( - loggerFactory, - trafficSignal: this.isConnectedSignal, - orchestrationService: service, - taskExecutor: this); - - this.hostLifetime = hostApplicationLifetime; - this.options = options; - this.hostLifetime.ApplicationStarted.Register(this.OnApplicationStarted); - this.hostLifetime.ApplicationStopping.Register(this.OnApplicationStopping); - } - - async void OnApplicationStarted() - { - if (this.options.Value.Mode == TaskHubGrpcServerMode.ApiServerAndDispatcher) - { - // Wait for a client connection to be established before starting the dispatcher host. - // This ensures we don't do any wasteful polling of resources if no clients are available to process events. - await this.WaitForWorkItemClientConnection(); - await this.dispatcherHost.StartAsync(this.hostLifetime.ApplicationStopping); - } - } - - async void OnApplicationStopping() - { - if (this.options.Value.Mode == TaskHubGrpcServerMode.ApiServerAndDispatcher) - { - // Give a maximum of 60 minutes for outstanding tasks to complete. - // REVIEW: Is this enough? What if there's an activity job that takes 4 hours to complete? Should this be configurable? - using CancellationTokenSource timeoutCts = new(TimeSpan.FromMinutes(60)); - await this.dispatcherHost.StopAsync(timeoutCts.Token); - } - } - - /// - /// Blocks until a remote client calls the operation to start fetching work items. - /// - /// Returns a task that completes once a work-item client is connected. - async Task WaitForWorkItemClientConnection() - { - Stopwatch waitTimeStopwatch = Stopwatch.StartNew(); - TimeSpan logInterval = TimeSpan.FromMinutes(1); - - try - { - while (!await this.isConnectedSignal.WaitAsync(logInterval, this.hostLifetime.ApplicationStopping)) - { - this.log.WaitingForClientConnection(waitTimeStopwatch.Elapsed); - } - } - catch (OperationCanceledException) - { - // shutting down - } - } - - public override Task Hello(Empty request, ServerCallContext context) => Task.FromResult(new Empty()); - - public override Task CreateTaskHub(P.CreateTaskHubRequest request, ServerCallContext context) - { - this.service.CreateAsync(request.RecreateIfExists); - return Task.FromResult(new P.CreateTaskHubResponse()); - } - - public override Task DeleteTaskHub(P.DeleteTaskHubRequest request, ServerCallContext context) - { - this.service.DeleteAsync(); - return Task.FromResult(new P.DeleteTaskHubResponse()); - } - - public override async Task StartInstance(P.CreateInstanceRequest request, ServerCallContext context) - { - var instance = new OrchestrationInstance - { - InstanceId = request.InstanceId ?? Guid.NewGuid().ToString("N"), - ExecutionId = Guid.NewGuid().ToString(), - }; - - // TODO: Structured logging - this.log.LogInformation("Creating a new instance with ID = {instanceID}", instance.InstanceId); - - try - { - await this.client.CreateTaskOrchestrationAsync( - new TaskMessage - { - Event = new ExecutionStartedEvent(-1, request.Input) - { - Name = request.Name, - Version = request.Version, - OrchestrationInstance = instance, - Tags = request.Tags.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), - ParentTraceContext = request.ParentTraceContext is not null - ? new(request.ParentTraceContext.TraceParent, request.ParentTraceContext.TraceState) - : null - }, - OrchestrationInstance = instance, - }); - } - catch (Exception e) - { - // TODO: Structured logging - this.log.LogError(e, "An error occurred when trying to create a new instance"); - throw; - } - - return new P.CreateInstanceResponse - { - InstanceId = instance.InstanceId, - }; - } - - public override async Task RaiseEvent(P.RaiseEventRequest request, ServerCallContext context) - { - try - { - await this.client.SendTaskOrchestrationMessageAsync( - new TaskMessage - { - Event = new EventRaisedEvent(-1, request.Input) - { - Name = request.Name, - }, - OrchestrationInstance = new OrchestrationInstance - { - InstanceId = request.InstanceId, - }, - }); - } - catch (Exception e) - { - // TODO: Structured logging - this.log.LogError(e, "An error occurred when trying to raise an event."); - throw; - } - - // No fields in the response - return new P.RaiseEventResponse(); - } - - public override async Task TerminateInstance(P.TerminateRequest request, ServerCallContext context) - { - try - { - await this.client.ForceTerminateTaskOrchestrationAsync( - request.InstanceId, - request.Output); - } - catch (Exception e) - { - // TODO: Structured logging - this.log.LogError(e, "An error occurred when trying to terminate an instance."); - throw; - } - - // No fields in the response - return new P.TerminateResponse(); - } - - public override async Task GetInstance(P.GetInstanceRequest request, ServerCallContext context) - { - OrchestrationState state = await this.client.GetOrchestrationStateAsync(request.InstanceId, executionId: null); - if (state == null) - { - return new P.GetInstanceResponse() { Exists = false }; - } - - return CreateGetInstanceResponse(state, request); - } - - public override async Task QueryInstances(P.QueryInstancesRequest request, ServerCallContext context) - { - if (this.client is IOrchestrationServiceQueryClient queryClient) - { - OrchestrationQuery query = ProtobufUtils.ToOrchestrationQuery(request); - OrchestrationQueryResult result = await queryClient.GetOrchestrationWithQueryAsync(query, context.CancellationToken); - return ProtobufUtils.CreateQueryInstancesResponse(result, request); - } - else - { - throw new NotSupportedException($"{this.client.GetType().Name} doesn't support query operations."); - } - } - - public override async Task PurgeInstances(P.PurgeInstancesRequest request, ServerCallContext context) - { - if (this.client is IOrchestrationServicePurgeClient purgeClient) - { - PurgeResult result; - switch (request.RequestCase) - { - case P.PurgeInstancesRequest.RequestOneofCase.InstanceId: - result = await purgeClient.PurgeInstanceStateAsync(request.InstanceId); - break; - - case P.PurgeInstancesRequest.RequestOneofCase.PurgeInstanceFilter: - PurgeInstanceFilter purgeInstanceFilter = ProtobufUtils.ToPurgeInstanceFilter(request); - result = await purgeClient.PurgeInstanceStateAsync(purgeInstanceFilter); - break; - - default: - throw new ArgumentException($"Unknown purge request type '{request.RequestCase}'."); - } - return ProtobufUtils.CreatePurgeInstancesResponse(result); - } - else - { - throw new NotSupportedException($"{this.client.GetType().Name} doesn't support purge operations."); - } - } - - public override async Task WaitForInstanceStart(P.GetInstanceRequest request, ServerCallContext context) - { - while (true) - { - // Keep fetching the status until we get one of the states we care about - OrchestrationState state = await this.client.GetOrchestrationStateAsync(request.InstanceId, executionId: null); - if (state != null && state.OrchestrationStatus != OrchestrationStatus.Pending) - { - return CreateGetInstanceResponse(state, request); - } - - // TODO: Backoff strategy if we're delaying for a long time. - // The cancellation token is what will break us out of this loop if the orchestration - // never leaves the "Pending" state. - await Task.Delay(TimeSpan.FromMilliseconds(500), context.CancellationToken); - } - } - - public override async Task WaitForInstanceCompletion(P.GetInstanceRequest request, ServerCallContext context) - { - OrchestrationState state = await this.client.WaitForOrchestrationAsync( - request.InstanceId, - executionId: null, - timeout: Timeout.InfiniteTimeSpan, - context.CancellationToken); - - return CreateGetInstanceResponse(state, request); - } - - static P.GetInstanceResponse CreateGetInstanceResponse(OrchestrationState state, P.GetInstanceRequest request) - { - return new P.GetInstanceResponse - { - Exists = true, - OrchestrationState = new P.OrchestrationState - { - InstanceId = state.OrchestrationInstance.InstanceId, - Name = state.Name, - OrchestrationStatus = (P.OrchestrationStatus)state.OrchestrationStatus, - CreatedTimestamp = Timestamp.FromDateTime(state.CreatedTime), - LastUpdatedTimestamp = Timestamp.FromDateTime(state.LastUpdatedTime), - Input = request.GetInputsAndOutputs ? state.Input : null, - Output = request.GetInputsAndOutputs ? state.Output : null, - CustomStatus = request.GetInputsAndOutputs ? state.Status : null, - FailureDetails = request.GetInputsAndOutputs ? ProtobufUtils.GetFailureDetails(state.FailureDetails) : null, - Tags = { state.Tags } - } - }; - } - - public override async Task SuspendInstance(P.SuspendRequest request, ServerCallContext context) - { - TaskMessage taskMessage = new() - { - OrchestrationInstance = new OrchestrationInstance { InstanceId = request.InstanceId }, - Event = new ExecutionSuspendedEvent(-1, request.Reason), - }; - - await this.client.SendTaskOrchestrationMessageAsync(taskMessage); - return new P.SuspendResponse(); - } - - public override async Task ResumeInstance(P.ResumeRequest request, ServerCallContext context) - { - TaskMessage taskMessage = new() - { - OrchestrationInstance = new OrchestrationInstance { InstanceId = request.InstanceId }, - Event = new ExecutionResumedEvent(-1, request.Reason), - }; - - await this.client.SendTaskOrchestrationMessageAsync(taskMessage); - return new P.ResumeResponse(); - } - - public override async Task RestartInstance(P.RestartInstanceRequest request, ServerCallContext context) - { - try - { - // Get the original orchestration state - IList states = await this.client.GetOrchestrationStateAsync(request.InstanceId, false); - - if (states == null || states.Count == 0) - { - throw new RpcException(new Status(StatusCode.NotFound, $"An orchestration with the instanceId {request.InstanceId} was not found.")); - } - - OrchestrationState state = states[0]; - - // Check if the state is null - if (state == null) - { - throw new RpcException(new Status(StatusCode.NotFound, $"An orchestration with the instanceId {request.InstanceId} was not found.")); - } - - string newInstanceId = request.RestartWithNewInstanceId ? Guid.NewGuid().ToString("N") : request.InstanceId; - - // Create a new orchestration instance - OrchestrationInstance newInstance = new() - { - InstanceId = newInstanceId, - ExecutionId = Guid.NewGuid().ToString("N"), - }; - - // Create an ExecutionStartedEvent with the original input - ExecutionStartedEvent startedEvent = new(-1, state.Input) - { - Name = state.Name, - Version = state.Version ?? string.Empty, - OrchestrationInstance = newInstance, - }; - - TaskMessage taskMessage = new() - { - OrchestrationInstance = newInstance, - Event = startedEvent, - }; - - await this.client.CreateTaskOrchestrationAsync(taskMessage); - - return new P.RestartInstanceResponse - { - InstanceId = newInstanceId, - }; - } - catch (RpcException) - { - // Re-throw RpcException as-is - throw; - } - catch (Exception ex) - { - this.log.LogError(ex, "Error restarting orchestration instance {InstanceId}", request.InstanceId); - throw new RpcException(new Status(StatusCode.Internal, ex.Message)); - } - } - - /// - /// Invoked by the remote SDK over gRPC when an orchestrator task (episode) is completed. - /// - /// Details about the orchestration execution, including the list of orchestrator actions. - /// Context for the server-side gRPC call. - /// Returns an empty ack back to the remote SDK that we've received the completion. - public override Task CompleteOrchestratorTask(P.OrchestratorResponse request, ServerCallContext context) - { - if (!this.pendingOrchestratorTasks.TryRemove( - request.InstanceId, - out TaskCompletionSource? tcs)) - { - // TODO: Log? - throw new RpcException(new Status(StatusCode.NotFound, $"Orchestration not found")); - } - - GrpcOrchestratorExecutionResult result = new() - { - Actions = request.Actions.Select(ProtobufUtils.ToOrchestratorAction), - CustomStatus = request.CustomStatus, - OrchestrationActivitySpanId = request.OrchestrationTraceContext?.SpanID, - OrchestrationActivityStartTime = request.OrchestrationTraceContext?.SpanStartTime?.ToDateTimeOffset(), - }; - - tcs.TrySetResult(result); - - return EmptyCompleteTaskResponse; - } - - /// - /// Invoked by the remote SDK over gRPC when an activity task (episode) is completed. - /// - /// Details about the completed activity task, including the output. - /// Context for the server-side gRPC call. - /// Returns an empty ack back to the remote SDK that we've received the completion. - public override Task CompleteActivityTask(P.ActivityResponse response, ServerCallContext context) - { - string taskIdKey = GetTaskIdKey(response.InstanceId, response.TaskId); - if (!this.pendingActivityTasks.TryRemove(taskIdKey, out TaskCompletionSource? tcs)) - { - // TODO: Log? - throw new RpcException(new Status(StatusCode.NotFound, $"Activity not found")); - } - - HistoryEvent resultEvent; - if (response.FailureDetails == null) - { - resultEvent = new TaskCompletedEvent(-1, response.TaskId, response.Result); - } - else - { - resultEvent = new TaskFailedEvent( - eventId: -1, - taskScheduledId: response.TaskId, - reason: null, - details: null, - failureDetails: ProtobufUtils.GetFailureDetails(response.FailureDetails)); - } - - tcs.TrySetResult(new ActivityExecutionResult { ResponseEvent = resultEvent }); - return EmptyCompleteTaskResponse; - } - - public override async Task GetWorkItems(P.GetWorkItemsRequest request, IServerStreamWriter responseStream, ServerCallContext context) - { - // Record whether the client supports history streaming - this.supportsHistoryStreaming = request.Capabilities.Contains(P.WorkerCapability.HistoryStreaming); - // Use a lock to mitigate the race condition where we signal the dispatch host to start but haven't - // yet saved a reference to the client response stream. - lock (this.isConnectedSignal) - { - int retryCount = 0; - while (!this.isConnectedSignal.Set()) - { - // Retries are needed when a client (like a test suite) connects and disconnects rapidly, causing a race - // condition where we don't reset the signal quickly enough to avoid ResourceExausted errors. - if (retryCount <= 5) - { - Thread.Sleep(10); // Can't use await inside the body of a lock statement so we have to block the thread - } - else - { - throw new RpcException(new Status(StatusCode.ResourceExhausted, "Another client is already connected")); - } - } - - this.log.ClientConnected(context.Peer, context.Deadline); - this.workerToClientStream = responseStream; - } - - try - { - await Task.Delay(Timeout.InfiniteTimeSpan, context.CancellationToken); - } - catch (OperationCanceledException) - { - this.log.ClientDisconnected(context.Peer); - } - finally - { - // Resetting this signal causes the dispatchers to stop fetching new work. - this.isConnectedSignal.Reset(); - - // Transition back to the "waiting for connection" state. - // This background task is just to log "waiting for connection" messages. - _ = Task.Run(this.WaitForWorkItemClientConnection); - } - } - - public override async Task StreamInstanceHistory(P.StreamInstanceHistoryRequest request, IServerStreamWriter responseStream, ServerCallContext context) - { - if (this.streamingPastEvents.TryGetValue(request.InstanceId, out List? pastEvents)) - { - const int MaxChunkBytes = 256 * 1024; // 256KB per chunk to simulate chunked streaming - int currentSize = 0; - P.HistoryChunk chunk = new(); - - foreach (P.HistoryEvent e in pastEvents) - { - int eventSize = e.CalculateSize(); - if (currentSize > 0 && currentSize + eventSize > MaxChunkBytes) - { - await responseStream.WriteAsync(chunk); - chunk = new P.HistoryChunk(); - currentSize = 0; - } - - chunk.Events.Add(e); - currentSize += eventSize; - } - - if (chunk.Events.Count > 0) - { - await responseStream.WriteAsync(chunk); - } - } - } - - /// - /// Invoked by the when a work item is available, proxies the call to execute an orchestrator over a gRPC channel. - /// - /// - async Task ITaskExecutor.ExecuteOrchestrator( - OrchestrationInstance instance, - IEnumerable pastEvents, - IEnumerable newEvents) - { - var executionStartedEvent = pastEvents.OfType().FirstOrDefault(); - - P.OrchestrationTraceContext? orchestrationTraceContext = executionStartedEvent?.ParentTraceContext?.SpanId is not null - ? new P.OrchestrationTraceContext - { - SpanID = executionStartedEvent.ParentTraceContext.SpanId, - SpanStartTime = executionStartedEvent.ParentTraceContext.ActivityStartTime?.ToTimestamp(), - } - : null; - - // Create a task completion source that represents the async completion of the orchestrator execution. - // This must be done before we start the orchestrator execution. - TaskCompletionSource tcs = - this.CreateTaskCompletionSourceForOrchestrator(instance.InstanceId); - - try - { - var orkRequest = new P.OrchestratorRequest - { - InstanceId = instance.InstanceId, - ExecutionId = instance.ExecutionId, - NewEvents = { newEvents.Select(ProtobufUtils.ToHistoryEventProto) }, - OrchestrationTraceContext = orchestrationTraceContext, - }; - - // Decide whether to stream based on total size of past events (> 1MiB) - List protoPastEvents = pastEvents.Select(ProtobufUtils.ToHistoryEventProto).ToList(); - int totalBytes = 0; - foreach (P.HistoryEvent ev in protoPastEvents) - { - totalBytes += ev.CalculateSize(); - } - - if (this.supportsHistoryStreaming && totalBytes > (1024)) - { - orkRequest.RequiresHistoryStreaming = true; - // Store past events to serve via StreamInstanceHistory - this.streamingPastEvents[instance.InstanceId] = protoPastEvents; - } - else - { - // Embed full history in the work item - orkRequest.PastEvents.AddRange(protoPastEvents); - } - - await this.SendWorkItemToClientAsync(new P.WorkItem - { - OrchestratorRequest = orkRequest, - }); - } - catch - { - // Remove the TaskCompletionSource that we just created - this.RemoveOrchestratorTaskCompletionSource(instance.InstanceId); - throw; - } - - // The TCS will be completed on the message stream handler when it gets a response back from the remote process - // TODO: How should we handle timeouts if the remote process never sends a response? - // Probably need to have a static timeout (e.g. 5 minutes). - return await tcs.Task; - } - - async Task ITaskExecutor.ExecuteActivity(OrchestrationInstance instance, TaskScheduledEvent activityEvent) - { - // Create a task completion source that represents the async completion of the activity. - // This must be done before we start the activity execution. - TaskCompletionSource tcs = this.CreateTaskCompletionSourceForActivity( - instance.InstanceId, - activityEvent.EventId); - - try - { - await this.SendWorkItemToClientAsync(new P.WorkItem - { - ActivityRequest = new P.ActivityRequest - { - Name = activityEvent.Name, - Version = activityEvent.Version, - Input = activityEvent.Input, - TaskId = activityEvent.EventId, - OrchestrationInstance = new P.OrchestrationInstance - { - InstanceId = instance.InstanceId, - ExecutionId = instance.ExecutionId, - }, - ParentTraceContext = activityEvent.ParentTraceContext is not null - ? new() - { - TraceParent = activityEvent.ParentTraceContext.TraceParent, - TraceState = activityEvent.ParentTraceContext.TraceState, - } - : null, - } - }); - } - catch - { - // Remove the TaskCompletionSource that we just created - this.RemoveActivityTaskCompletionSource(instance.InstanceId, activityEvent.EventId); - throw; - } - - // The TCS will be completed on the message stream handler when it gets a response back from the remote process. - // TODO: How should we handle timeouts if the remote process never sends a response? - // Probably need a timeout feature for activities and/or a heartbeat API that activities - // can use to signal that they're still running. - return await tcs.Task; - } - - async Task SendWorkItemToClientAsync(P.WorkItem workItem) - { - IServerStreamWriter outputStream; - - // Use a lock to mitigate the race condition where we signal the dispatch host to start but haven't - // yet saved a reference to the client response stream. - lock (this.isConnectedSignal) - { - outputStream = this.workerToClientStream ?? - throw new Exception("TODO: No client is connected! Need to wait until a client connects before executing!"); - } - - // The gRPC channel can only handle one message at a time, so we need to serialize access to it. - await this.sendWorkItemLock.WaitAsync(); - try - { - await outputStream.WriteAsync(workItem); - } - finally - { - this.sendWorkItemLock.Release(); - } - } - - TaskCompletionSource CreateTaskCompletionSourceForOrchestrator(string instanceId) - { - TaskCompletionSource tcs = new(); - this.pendingOrchestratorTasks.TryAdd(instanceId, tcs); - return tcs; - } - - void RemoveOrchestratorTaskCompletionSource(string instanceId) - { - this.pendingOrchestratorTasks.TryRemove(instanceId, out _); - } - - TaskCompletionSource CreateTaskCompletionSourceForActivity(string instanceId, int taskId) - { - string taskIdKey = GetTaskIdKey(instanceId, taskId); - TaskCompletionSource tcs = new(); - this.pendingActivityTasks.TryAdd(taskIdKey, tcs); - return tcs; - } - - void RemoveActivityTaskCompletionSource(string instanceId, int taskId) - { - string taskIdKey = GetTaskIdKey(instanceId, taskId); - this.pendingActivityTasks.TryRemove(taskIdKey, out _); - } - - static string GetTaskIdKey(string instanceId, int taskId) - { - return string.Concat(instanceId, "__", taskId.ToString()); - } - - public override Task AbandonTaskActivityWorkItem(P.AbandonActivityTaskRequest request, ServerCallContext context) - { - return Task.FromResult(new()); - } - - public override Task AbandonTaskOrchestratorWorkItem(P.AbandonOrchestrationTaskRequest request, ServerCallContext context) - { - return Task.FromResult(new()); - } - - /// - /// A implementation that is used to control whether the task hub - /// dispatcher can fetch new work-items, based on whether a client is currently connected. - /// - class IsConnectedSignal : ITrafficSignal - { - readonly AsyncManualResetEvent isConnectedEvent = new(isSignaled: false); - - /// - public string WaitReason => "Waiting for a client to connect"; - - /// - /// Blocks the caller until the method is called, which means a client is connected. - /// - /// - public Task WaitAsync(TimeSpan waitTime, CancellationToken cancellationToken) - { - return this.isConnectedEvent.WaitAsync(waitTime, cancellationToken); - } - - /// - /// Signals the dispatchers to start fetching new work-items. - /// - /// - /// Returns true if the current thread transitioned the event to the "signaled" state; - /// otherwise false, meaning some other thread already called on this signal. - /// - public bool Set() => this.isConnectedEvent.Set(); - - /// - /// Causes the dispatchers to stop fetching new work-items. - /// - public void Reset() => this.isConnectedEvent.Reset(); - } -} +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Globalization; +using DurableTask.Core; +using DurableTask.Core.History; +using DurableTask.Core.Query; +using Google.Protobuf.WellKnownTypes; +using Grpc.Core; +using Microsoft.DurableTask.Testing.Sidecar.Dispatcher; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using P = Microsoft.DurableTask.Protobuf; + +namespace Microsoft.DurableTask.Testing.Sidecar.Grpc; + +/// +/// gRPC server implementation for the TaskHub sidecar service. +/// +/// +/// This class implements the gRPC service that handles communication between the durable task worker +/// and the orchestration service, managing work items and execution results. +/// +public class TaskHubGrpcServer : P.TaskHubSidecarService.TaskHubSidecarServiceBase, ITaskExecutor, IDisposable // CA1001: Types owning disposable fields should be disposable +{ + static readonly Task EmptyCompleteTaskResponse = Task.FromResult(new P.CompleteTaskResponse()); + + readonly ConcurrentDictionary> pendingOrchestratorTasks = new(StringComparer.OrdinalIgnoreCase); + readonly ConcurrentDictionary> pendingActivityTasks = new(StringComparer.OrdinalIgnoreCase); + + readonly ILogger log; + readonly IOrchestrationService service; + readonly IOrchestrationServiceClient client; + readonly IHostApplicationLifetime hostLifetime; + readonly IOptions options; + readonly TaskHubDispatcherHost dispatcherHost; + readonly IsConnectedSignal isConnectedSignal = new(); + readonly SemaphoreSlim sendWorkItemLock = new(initialCount: 1); + readonly ConcurrentDictionary> streamingPastEvents = new(StringComparer.OrdinalIgnoreCase); + + volatile bool supportsHistoryStreaming; + + // Initialized when a client connects to this service to receive work-item commands. + IServerStreamWriter? workerToClientStream; + + bool disposed; + + /// + /// Initializes a new instance of the class. + /// + /// The host application lifetime. + /// The logger factory. + /// The orchestration service. + /// The orchestration service client. + /// The server options. + public TaskHubGrpcServer( + IHostApplicationLifetime hostApplicationLifetime, + ILoggerFactory loggerFactory, + IOrchestrationService service, + IOrchestrationServiceClient client, + IOptions options) + { + ArgumentNullException.ThrowIfNull(hostApplicationLifetime, nameof(hostApplicationLifetime)); + ArgumentNullException.ThrowIfNull(loggerFactory, nameof(loggerFactory)); + ArgumentNullException.ThrowIfNull(service, nameof(service)); + ArgumentNullException.ThrowIfNull(client, nameof(client)); + ArgumentNullException.ThrowIfNull(options, nameof(options)); + + this.service = service; + this.client = client; + this.log = loggerFactory.CreateLogger("Microsoft.DurableTask.Sidecar"); + this.dispatcherHost = new TaskHubDispatcherHost( + loggerFactory, + trafficSignal: this.isConnectedSignal, + orchestrationService: service, + taskExecutor: this); + + this.hostLifetime = hostApplicationLifetime; + this.options = options; + this.hostLifetime.ApplicationStarted.Register(this.OnApplicationStarted); + this.hostLifetime.ApplicationStopping.Register(this.OnApplicationStopping); + } + + /// + /// Disposes the server resources. + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Disposes the server resources. + /// + /// Whether disposing from Dispose method. + protected virtual void Dispose(bool disposing) + { + if (!this.disposed && disposing) + { + this.sendWorkItemLock?.Dispose(); + this.isConnectedSignal?.Reset(); // Clean up the signal + this.disposed = true; + } + } + + async void OnApplicationStarted() + { + if (this.options.Value.Mode == TaskHubGrpcServerMode.ApiServerAndDispatcher) + { + // Wait for a client connection to be established before starting the dispatcher host. + // This ensures we don't do any wasteful polling of resources if no clients are available to process events. + await this.WaitForWorkItemClientConnection(); + await this.dispatcherHost.StartAsync(this.hostLifetime.ApplicationStopping); + } + } + + async void OnApplicationStopping() + { + if (this.options.Value.Mode == TaskHubGrpcServerMode.ApiServerAndDispatcher) + { + // Give a maximum of 60 minutes for outstanding tasks to complete. + // REVIEW: Is this enough? What if there's an activity job that takes 4 hours to complete? Should this be configurable? + using CancellationTokenSource timeoutCts = new(TimeSpan.FromMinutes(60)); + await this.dispatcherHost.StopAsync(timeoutCts.Token); + } + } + + /// + /// Blocks until a remote client calls the operation to start fetching work items. + /// + /// Returns a task that completes once a work-item client is connected. + async Task WaitForWorkItemClientConnection() + { + Stopwatch waitTimeStopwatch = Stopwatch.StartNew(); + TimeSpan logInterval = TimeSpan.FromMinutes(1); + + try + { + while (!await this.isConnectedSignal.WaitAsync(logInterval, this.hostLifetime.ApplicationStopping)) + { + this.log.WaitingForClientConnection(waitTimeStopwatch.Elapsed); + } + } + catch (OperationCanceledException) + { + // shutting down + } + } + + /// + /// Handles the Hello gRPC call. + /// + /// The empty request. + /// The server call context. + /// An empty response. + public override Task Hello(Empty request, ServerCallContext context) => Task.FromResult(new Empty()); + + /// + /// Creates a task hub. + /// + /// The create task hub request. + /// The server call context. + /// A create task hub response. + public override Task CreateTaskHub(P.CreateTaskHubRequest request, ServerCallContext context) + { + this.service.CreateAsync(request.RecreateIfExists); + return Task.FromResult(new P.CreateTaskHubResponse()); + } + + /// + /// Deletes a task hub. + /// + /// The delete task hub request. + /// The server call context. + /// A delete task hub response. + public override Task DeleteTaskHub(P.DeleteTaskHubRequest request, ServerCallContext context) + { + this.service.DeleteAsync(); + return Task.FromResult(new P.DeleteTaskHubResponse()); + } + + /// + /// Starts a new orchestration instance. + /// + /// The create instance request. + /// The server call context. + /// A create instance response. + public override async Task StartInstance(P.CreateInstanceRequest request, ServerCallContext context) + { + var instance = new OrchestrationInstance + { + InstanceId = request.InstanceId ?? Guid.NewGuid().ToString("N"), + ExecutionId = Guid.NewGuid().ToString(), + }; + + // TODO: Structured logging + this.log.LogInformation("Creating a new instance with ID = {InstanceId}", instance.InstanceId); + + try + { + await this.client.CreateTaskOrchestrationAsync( + new TaskMessage + { + Event = new ExecutionStartedEvent(-1, request.Input) + { + Name = request.Name, + Version = request.Version, + OrchestrationInstance = instance, + Tags = request.Tags.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), + ParentTraceContext = request.ParentTraceContext is not null + ? new(request.ParentTraceContext.TraceParent, request.ParentTraceContext.TraceState) + : null + }, + OrchestrationInstance = instance, + }); + } + catch (Exception e) + { + // TODO: Structured logging + this.log.LogError(e, "An error occurred when trying to create a new instance"); + throw; + } + + return new P.CreateInstanceResponse + { + InstanceId = instance.InstanceId, + }; + } + + /// + /// Raises an event to an orchestration instance. + /// + /// The raise event request. + /// The server call context. + /// A raise event response. + public override async Task RaiseEvent(P.RaiseEventRequest request, ServerCallContext context) + { + try + { + await this.client.SendTaskOrchestrationMessageAsync( + new TaskMessage + { + Event = new EventRaisedEvent(-1, request.Input) + { + Name = request.Name, + }, + OrchestrationInstance = new OrchestrationInstance + { + InstanceId = request.InstanceId, + }, + }); + } + catch (Exception e) + { + // TODO: Structured logging + this.log.LogError(e, "An error occurred when trying to raise an event."); + throw; + } + + // No fields in the response + return new P.RaiseEventResponse(); + } + + /// + /// Terminates an orchestration instance. + /// + /// The terminate request. + /// The server call context. + /// A terminate response. + public override async Task TerminateInstance(P.TerminateRequest request, ServerCallContext context) + { + try + { + await this.client.ForceTerminateTaskOrchestrationAsync( + request.InstanceId, + request.Output); + } + catch (Exception e) + { + // TODO: Structured logging + this.log.LogError(e, "An error occurred when trying to terminate an instance."); + throw; + } + + // No fields in the response + return new P.TerminateResponse(); + } + + /// + /// Gets an orchestration instance. + /// + /// The get instance request. + /// The server call context. + /// A get instance response. + public override async Task GetInstance(P.GetInstanceRequest request, ServerCallContext context) + { + OrchestrationState state = await this.client.GetOrchestrationStateAsync(request.InstanceId, executionId: null); + if (state == null) + { + return new P.GetInstanceResponse() { Exists = false }; + } + + return CreateGetInstanceResponse(state, request); + } + + /// + /// Queries orchestration instances. + /// + /// The query instances request. + /// The server call context. + /// A query instances response. + public override async Task QueryInstances(P.QueryInstancesRequest request, ServerCallContext context) + { + if (this.client is IOrchestrationServiceQueryClient queryClient) + { + OrchestrationQuery query = ProtobufUtils.ToOrchestrationQuery(request); + OrchestrationQueryResult result = await queryClient.GetOrchestrationWithQueryAsync(query, context.CancellationToken); + return ProtobufUtils.CreateQueryInstancesResponse(result, request); + } + else + { + throw new NotSupportedException($"{this.client.GetType().Name} doesn't support query operations."); + } + } + + /// + /// Purges orchestration instances. + /// + /// The purge instances request. + /// The server call context. + /// A purge instances response. + public override async Task PurgeInstances(P.PurgeInstancesRequest request, ServerCallContext context) + { + if (this.client is IOrchestrationServicePurgeClient purgeClient) + { + PurgeResult result; + switch (request.RequestCase) + { + case P.PurgeInstancesRequest.RequestOneofCase.InstanceId: + result = await purgeClient.PurgeInstanceStateAsync(request.InstanceId); + break; + + case P.PurgeInstancesRequest.RequestOneofCase.PurgeInstanceFilter: + PurgeInstanceFilter purgeInstanceFilter = ProtobufUtils.ToPurgeInstanceFilter(request); + result = await purgeClient.PurgeInstanceStateAsync(purgeInstanceFilter); + break; + + default: + // CA2201: Use specific exception types + throw new NotSupportedException($"Unknown purge request type '{request.RequestCase}'."); + } + return ProtobufUtils.CreatePurgeInstancesResponse(result); + } + else + { + throw new NotSupportedException($"{this.client.GetType().Name} doesn't support purge operations."); + } + } + + /// + /// Waits for an orchestration instance to start. + /// + /// The get instance request. + /// The server call context. + /// A get instance response. + public override async Task WaitForInstanceStart(P.GetInstanceRequest request, ServerCallContext context) + { + while (true) + { + // Keep fetching the status until we get one of the states we care about + OrchestrationState state = await this.client.GetOrchestrationStateAsync(request.InstanceId, executionId: null); + if (state != null && state.OrchestrationStatus != OrchestrationStatus.Pending) + { + return CreateGetInstanceResponse(state, request); + } + + // TODO: Backoff strategy if we're delaying for a long time. + // The cancellation token is what will break us out of this loop if the orchestration + // never leaves the "Pending" state. + await Task.Delay(TimeSpan.FromMilliseconds(500), context.CancellationToken); + } + } + + /// + /// Waits for an orchestration instance to complete. + /// + /// The get instance request. + /// The server call context. + /// A get instance response. + public override async Task WaitForInstanceCompletion(P.GetInstanceRequest request, ServerCallContext context) + { + OrchestrationState state = await this.client.WaitForOrchestrationAsync( + request.InstanceId, + executionId: null, + timeout: Timeout.InfiniteTimeSpan, + context.CancellationToken); + + return CreateGetInstanceResponse(state, request); + } + + static P.GetInstanceResponse CreateGetInstanceResponse(OrchestrationState state, P.GetInstanceRequest request) + { + return new P.GetInstanceResponse + { + Exists = true, + OrchestrationState = new P.OrchestrationState + { + InstanceId = state.OrchestrationInstance.InstanceId, + Name = state.Name, + OrchestrationStatus = (P.OrchestrationStatus)state.OrchestrationStatus, + CreatedTimestamp = Timestamp.FromDateTime(state.CreatedTime), + LastUpdatedTimestamp = Timestamp.FromDateTime(state.LastUpdatedTime), + Input = request.GetInputsAndOutputs ? state.Input : null, + Output = request.GetInputsAndOutputs ? state.Output : null, + CustomStatus = request.GetInputsAndOutputs ? state.Status : null, + FailureDetails = request.GetInputsAndOutputs ? ProtobufUtils.GetFailureDetails(state.FailureDetails) : null, + Tags = { state.Tags }, + }, + }; + } + + /// + /// Suspends an orchestration instance. + /// + /// The suspend request. + /// The server call context. + /// A suspend response. + public override async Task SuspendInstance(P.SuspendRequest request, ServerCallContext context) + { + TaskMessage taskMessage = new() + { + OrchestrationInstance = new OrchestrationInstance { InstanceId = request.InstanceId }, + Event = new ExecutionSuspendedEvent(-1, request.Reason), + }; + + await this.client.SendTaskOrchestrationMessageAsync(taskMessage); + return new P.SuspendResponse(); + } + + /// + /// Resumes an orchestration instance. + /// + /// The resume request. + /// The server call context. + /// A resume response. + public override async Task ResumeInstance(P.ResumeRequest request, ServerCallContext context) + { + TaskMessage taskMessage = new() + { + OrchestrationInstance = new OrchestrationInstance { InstanceId = request.InstanceId }, + Event = new ExecutionResumedEvent(-1, request.Reason), + }; + + await this.client.SendTaskOrchestrationMessageAsync(taskMessage); + return new P.ResumeResponse(); + } + + /// + /// Restarts an orchestration instance. + /// + /// The restart instance request. + /// The server call context. + /// A restart instance response. + public override async Task RestartInstance(P.RestartInstanceRequest request, ServerCallContext context) + { + try + { + // Get the original orchestration state + IList states = await this.client.GetOrchestrationStateAsync(request.InstanceId, false); + + if (states == null || states.Count == 0) + { + throw new RpcException(new Status(StatusCode.NotFound, $"An orchestration with the instanceId {request.InstanceId} was not found.")); + } + + OrchestrationState state = states[0]; + + // Check if the state is null + if (state == null) + { + throw new RpcException(new Status(StatusCode.NotFound, $"An orchestration with the instanceId {request.InstanceId} was not found.")); + } + + string newInstanceId = request.RestartWithNewInstanceId ? Guid.NewGuid().ToString("N") : request.InstanceId; + + // Create a new orchestration instance + OrchestrationInstance newInstance = new() + { + InstanceId = newInstanceId, + ExecutionId = Guid.NewGuid().ToString("N"), + }; + + // Create an ExecutionStartedEvent with the original input + ExecutionStartedEvent startedEvent = new(-1, state.Input) + { + Name = state.Name, + Version = state.Version ?? string.Empty, + OrchestrationInstance = newInstance, + }; + + TaskMessage taskMessage = new() + { + OrchestrationInstance = newInstance, + Event = startedEvent, + }; + + await this.client.CreateTaskOrchestrationAsync(taskMessage); + + return new P.RestartInstanceResponse + { + InstanceId = newInstanceId, + }; + } + catch (RpcException) + { + // Re-throw RpcException as-is + throw; + } + catch (Exception ex) + { + this.log.LogError(ex, "Error restarting orchestration instance {InstanceId}", request.InstanceId); + throw new RpcException(new Status(StatusCode.Internal, ex.Message)); + } + } + + /// + /// Invoked by the remote SDK over gRPC when an orchestrator task (episode) is completed. + /// + /// Details about the orchestration execution, including the list of orchestrator actions. + /// Context for the server-side gRPC call. + /// Returns an empty ack back to the remote SDK that we've received the completion. + public override Task CompleteOrchestratorTask(P.OrchestratorResponse request, ServerCallContext context) + { + if (!this.pendingOrchestratorTasks.TryRemove( + request.InstanceId, + out TaskCompletionSource? tcs)) + { + // TODO: Log? + // CA2201: Use specific exception types + throw new RpcException(new Status(StatusCode.NotFound, $"Orchestration not found")); + } + + GrpcOrchestratorExecutionResult result = new() + { + Actions = request.Actions.Select(ProtobufUtils.ToOrchestratorAction), + CustomStatus = request.CustomStatus, + OrchestrationActivitySpanId = request.OrchestrationTraceContext?.SpanID, + OrchestrationActivityStartTime = request.OrchestrationTraceContext?.SpanStartTime?.ToDateTimeOffset(), + }; + + tcs.TrySetResult(result); + + return EmptyCompleteTaskResponse; + } + + /// + /// Invoked by the remote SDK over gRPC when an activity task (episode) is completed. + /// + /// Details about the completed activity task, including the output. + /// Context for the server-side gRPC call. + /// Returns an empty ack back to the remote SDK that we've received the completion. + public override Task CompleteActivityTask(P.ActivityResponse request, ServerCallContext context) + { + string taskIdKey = GetTaskIdKey(request.InstanceId, request.TaskId); + if (!this.pendingActivityTasks.TryRemove(taskIdKey, out TaskCompletionSource? tcs)) + { + // TODO: Log? + // CA2201: Use specific exception types + throw new RpcException(new Status(StatusCode.NotFound, $"Activity not found")); + } + + HistoryEvent resultEvent; + if (request.FailureDetails == null) + { + resultEvent = new TaskCompletedEvent(-1, request.TaskId, request.Result); + } + else + { + resultEvent = new TaskFailedEvent( + eventId: -1, + taskScheduledId: request.TaskId, + reason: null, + details: null, + failureDetails: ProtobufUtils.GetFailureDetails(request.FailureDetails)); + } + + tcs.TrySetResult(new ActivityExecutionResult { ResponseEvent = resultEvent }); + return EmptyCompleteTaskResponse; + } + + /// + /// Gets work items from the server. + /// + /// The get work items request. + /// The response stream. + /// The server call context. + /// A task representing the asynchronous operation. + public override async Task GetWorkItems(P.GetWorkItemsRequest request, IServerStreamWriter responseStream, ServerCallContext context) + { + // Record whether the client supports history streaming + this.supportsHistoryStreaming = request.Capabilities.Contains(P.WorkerCapability.HistoryStreaming); + // Use a lock to mitigate the race condition where we signal the dispatch host to start but haven't + // yet saved a reference to the client response stream. + lock (this.isConnectedSignal) + { + int retryCount = 0; + while (!this.isConnectedSignal.Set()) + { + // Retries are needed when a client (like a test suite) connects and disconnects rapidly, causing a race + // condition where we don't reset the signal quickly enough to avoid ResourceExausted errors. + if (retryCount <= 5) + { + Thread.Sleep(10); // Can't use await inside the body of a lock statement so we have to block the thread + } + else + { + throw new RpcException(new Status(StatusCode.ResourceExhausted, "Another client is already connected")); + } + retryCount++; // Fix the increment + } + + this.log.ClientConnected(context.Peer, context.Deadline); + this.workerToClientStream = responseStream; + } + + try + { + await Task.Delay(Timeout.InfiniteTimeSpan, context.CancellationToken); + } + catch (OperationCanceledException) + { + this.log.ClientDisconnected(context.Peer); + } + finally + { + // Resetting this signal causes the dispatchers to stop fetching new work. + this.isConnectedSignal.Reset(); + + // Transition back to the "waiting for connection" state. + // This background task is just to log "waiting for connection" messages. + _ = Task.Run(this.WaitForWorkItemClientConnection); + } + } + + public override async Task StreamInstanceHistory(P.StreamInstanceHistoryRequest request, IServerStreamWriter responseStream, ServerCallContext context) + { + if (this.streamingPastEvents.TryGetValue(request.InstanceId, out List? pastEvents)) + { + const int MaxChunkBytes = 256 * 1024; // 256KB per chunk to simulate chunked streaming + int currentSize = 0; + P.HistoryChunk chunk = new(); + + foreach (P.HistoryEvent e in pastEvents) + { + int eventSize = e.CalculateSize(); + if (currentSize > 0 && currentSize + eventSize > MaxChunkBytes) + { + await responseStream.WriteAsync(chunk); + chunk = new P.HistoryChunk(); + currentSize = 0; + } + + chunk.Events.Add(e); + currentSize += eventSize; + } + + if (chunk.Events.Count > 0) + { + await responseStream.WriteAsync(chunk); + } + } + } + + /// + /// Invoked by the when a work item is available, proxies the call to execute an orchestrator over a gRPC channel. + /// + /// + async Task ITaskExecutor.ExecuteOrchestrator( + OrchestrationInstance instance, + IEnumerable pastEvents, + IEnumerable newEvents) + { + var executionStartedEvent = pastEvents.OfType().FirstOrDefault(); + + P.OrchestrationTraceContext? orchestrationTraceContext = executionStartedEvent?.ParentTraceContext?.SpanId is not null + ? new P.OrchestrationTraceContext + { + SpanID = executionStartedEvent.ParentTraceContext.SpanId, + SpanStartTime = executionStartedEvent.ParentTraceContext.ActivityStartTime?.ToTimestamp(), + } + : null; + + // Create a task completion source that represents the async completion of the orchestrator execution. + // This must be done before we start the orchestrator execution. + TaskCompletionSource tcs = + this.CreateTaskCompletionSourceForOrchestrator(instance.InstanceId); + + try + { + var orkRequest = new P.OrchestratorRequest + { + InstanceId = instance.InstanceId, + ExecutionId = instance.ExecutionId, + NewEvents = { newEvents.Select(ProtobufUtils.ToHistoryEventProto) }, + OrchestrationTraceContext = orchestrationTraceContext, + }; + + // Decide whether to stream based on total size of past events (> 1MiB) + List protoPastEvents = pastEvents.Select(ProtobufUtils.ToHistoryEventProto).ToList(); + int totalBytes = 0; + foreach (P.HistoryEvent ev in protoPastEvents) + { + totalBytes += ev.CalculateSize(); + } + + if (this.supportsHistoryStreaming && totalBytes > (1024)) + { + orkRequest.RequiresHistoryStreaming = true; + // Store past events to serve via StreamInstanceHistory + this.streamingPastEvents[instance.InstanceId] = protoPastEvents; + } + else + { + // Embed full history in the work item + orkRequest.PastEvents.AddRange(protoPastEvents); + } + + await this.SendWorkItemToClientAsync(new P.WorkItem + { + OrchestratorRequest = orkRequest, + }); + } + catch + { + // Remove the TaskCompletionSource that we just created + this.RemoveOrchestratorTaskCompletionSource(instance.InstanceId); + throw; + } + + // The TCS will be completed on the message stream handler when it gets a response back from the remote process + // TODO: How should we handle timeouts if the remote process never sends a response? + // Probably need to have a static timeout (e.g. 5 minutes). + return await tcs.Task; + } + + async Task ITaskExecutor.ExecuteActivity(OrchestrationInstance instance, TaskScheduledEvent activityEvent) + { + // Create a task completion source that represents the async completion of the activity. + // This must be done before we start the activity execution. + TaskCompletionSource tcs = this.CreateTaskCompletionSourceForActivity( + instance.InstanceId, + activityEvent.EventId); + + try + { + await this.SendWorkItemToClientAsync(new P.WorkItem + { + ActivityRequest = new P.ActivityRequest + { + Name = activityEvent.Name, + Version = activityEvent.Version, + Input = activityEvent.Input, + TaskId = activityEvent.EventId, + OrchestrationInstance = new P.OrchestrationInstance + { + InstanceId = instance.InstanceId, + ExecutionId = instance.ExecutionId, + }, + ParentTraceContext = activityEvent.ParentTraceContext is not null + ? new() + { + TraceParent = activityEvent.ParentTraceContext.TraceParent, + TraceState = activityEvent.ParentTraceContext.TraceState, + } + : null, + }, + }); + } + catch + { + // Remove the TaskCompletionSource that we just created + this.RemoveActivityTaskCompletionSource(instance.InstanceId, activityEvent.EventId); + throw; + } + + // The TCS will be completed on the message stream handler when it gets a response back from the remote process. + // TODO: How should we handle timeouts if the remote process never sends a response? + // Probably need a timeout feature for activities and/or a heartbeat API that activities + // can use to signal that they're still running. + return await tcs.Task; + } + + async Task SendWorkItemToClientAsync(P.WorkItem workItem) + { + IServerStreamWriter outputStream; + + // Use a lock to mitigate the race condition where we signal the dispatch host to start but haven't + // yet saved a reference to the client response stream. + lock (this.isConnectedSignal) + { + outputStream = this.workerToClientStream ?? + // CA2201: Use specific exception types + throw new InvalidOperationException("TODO: No client is connected! Need to wait until a client connects before executing!"); + } + + // The gRPC channel can only handle one message at a time, so we need to serialize access to it. + await this.sendWorkItemLock.WaitAsync(); + try + { + await outputStream.WriteAsync(workItem); + } + finally + { + this.sendWorkItemLock.Release(); + } + } + + TaskCompletionSource CreateTaskCompletionSourceForOrchestrator(string instanceId) + { + TaskCompletionSource tcs = new(); + this.pendingOrchestratorTasks.TryAdd(instanceId, tcs); + return tcs; + } + + void RemoveOrchestratorTaskCompletionSource(string instanceId) + { + this.pendingOrchestratorTasks.TryRemove(instanceId, out _); + } + + TaskCompletionSource CreateTaskCompletionSourceForActivity(string instanceId, int taskId) + { + string taskIdKey = GetTaskIdKey(instanceId, taskId); + TaskCompletionSource tcs = new(); + this.pendingActivityTasks.TryAdd(taskIdKey, tcs); + return tcs; + } + + void RemoveActivityTaskCompletionSource(string instanceId, int taskId) + { + string taskIdKey = GetTaskIdKey(instanceId, taskId); + this.pendingActivityTasks.TryRemove(taskIdKey, out _); + } + + static string GetTaskIdKey(string instanceId, int taskId) + { + return string.Concat(instanceId, "_", taskId.ToString(CultureInfo.InvariantCulture)); + } + + /// + /// Abandons a task activity work item. + /// + /// The abandon activity task request. + /// The server call context. + /// An abandon activity task response. + public override Task AbandonTaskActivityWorkItem(P.AbandonActivityTaskRequest request, ServerCallContext context) + { + return Task.FromResult(new()); + } + + /// + /// Abandons a task orchestration work item. + /// + /// The abandon orchestration task request. + /// The server call context. + /// An abandon orchestration task response. + public override Task AbandonTaskOrchestratorWorkItem(P.AbandonOrchestrationTaskRequest request, ServerCallContext context) + { + return Task.FromResult(new()); + } + + /// + /// A implementation that is used to control whether the task hub + /// dispatcher can fetch new work-items, based on whether a client is currently connected. + /// + class IsConnectedSignal : ITrafficSignal + { + readonly AsyncManualResetEvent isConnectedEvent = new(isSignaled: false); + + /// + public string WaitReason => "Waiting for a client to connect"; + + /// + /// Blocks the caller until the method is called, which means a client is connected. + /// + /// + public Task WaitAsync(TimeSpan waitTime, CancellationToken cancellationToken) + { + return this.isConnectedEvent.WaitAsync(waitTime, cancellationToken); + } + + /// + /// Signals the dispatchers to start fetching new work-items. + /// + /// + /// Returns true if the current thread transitioned the event to the "signaled" state; + /// otherwise false, meaning some other thread already called on this signal. + /// + public bool Set() => this.isConnectedEvent.Set(); + + /// + /// Causes the dispatchers to stop fetching new work-items. + /// + public void Reset() => this.isConnectedEvent.Reset(); + } +} diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServerOptions.cs b/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServerOptions.cs similarity index 93% rename from test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServerOptions.cs rename to src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServerOptions.cs index ffeeacd5..f5756223 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Grpc/TaskHubGrpcServerOptions.cs +++ b/src/InProcessTestHost/Sidecar/Grpc/TaskHubGrpcServerOptions.cs @@ -1,18 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -namespace Microsoft.DurableTask.Sidecar.Grpc; - -/// -/// Options for configuring the task hub gRPC server. -/// -public class TaskHubGrpcServerOptions -{ - /// - /// The high-level mode of operation for the gRPC server. - /// - public TaskHubGrpcServerMode Mode { get; set; } -} +namespace Microsoft.DurableTask.Testing.Sidecar.Grpc; /// /// A set of options that determine what capabilities are enabled for the gRPC server. @@ -28,4 +17,16 @@ public enum TaskHubGrpcServerMode /// The gRPC server handles management API requests but not orchestration dispatching. /// ApiServerOnly, -} \ No newline at end of file +} + +/// +/// Options for configuring the task hub gRPC server. +/// +public class TaskHubGrpcServerOptions +{ + /// + /// The high-level mode of operation for the gRPC server. + /// + public TaskHubGrpcServerMode Mode { get; set; } +} + diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/InMemoryOrchestrationService.cs b/src/InProcessTestHost/Sidecar/InMemoryOrchestrationService.cs similarity index 71% rename from test/Grpc.IntegrationTests/GrpcSidecar/InMemoryOrchestrationService.cs rename to src/InProcessTestHost/Sidecar/InMemoryOrchestrationService.cs index 3494c64b..eb8fd042 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/InMemoryOrchestrationService.cs +++ b/src/InProcessTestHost/Sidecar/InMemoryOrchestrationService.cs @@ -1,704 +1,936 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Collections.Concurrent; -using System.Diagnostics.CodeAnalysis; -using System.Text.Json.Nodes; -using System.Threading.Channels; -using DurableTask.Core; -using DurableTask.Core.Exceptions; -using DurableTask.Core.History; -using DurableTask.Core.Query; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; - -namespace Microsoft.DurableTask.Sidecar; - -public class InMemoryOrchestrationService : IOrchestrationService, IOrchestrationServiceClient, IOrchestrationServiceQueryClient, IOrchestrationServicePurgeClient -{ - readonly InMemoryQueue activityQueue = new(); - readonly InMemoryInstanceStore instanceStore; - readonly ILogger logger; - - public int TaskOrchestrationDispatcherCount => 1; - - public int TaskActivityDispatcherCount => 1; - - public int MaxConcurrentTaskOrchestrationWorkItems => Environment.ProcessorCount; - - public int MaxConcurrentTaskActivityWorkItems => Environment.ProcessorCount; - - public BehaviorOnContinueAsNew EventBehaviourForContinueAsNew => BehaviorOnContinueAsNew.Carryover; - - public InMemoryOrchestrationService(ILoggerFactory? loggerFactory = null) - { - this.logger = (loggerFactory ?? NullLoggerFactory.Instance).CreateLogger("Microsoft.DurableTask.Sidecar.InMemoryStorageProvider"); - this.instanceStore = new InMemoryInstanceStore(this.logger); - } - - public Task AbandonTaskActivityWorkItemAsync(TaskActivityWorkItem workItem) - { - this.logger.LogWarning("Abandoning task activity work item {id}", workItem.Id); - this.activityQueue.Enqueue(workItem.TaskMessage); - return Task.CompletedTask; - } - - public Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) - { - this.instanceStore.AbandonInstance(workItem.NewMessages); - return Task.CompletedTask; - } - - public Task CompleteTaskActivityWorkItemAsync(TaskActivityWorkItem workItem, TaskMessage responseMessage) - { - this.instanceStore.AddMessage(responseMessage); - return Task.CompletedTask; - } - - public Task CompleteTaskOrchestrationWorkItemAsync( - TaskOrchestrationWorkItem workItem, - OrchestrationRuntimeState newOrchestrationRuntimeState, - IList outboundMessages, - IList orchestratorMessages, - IList timerMessages, - TaskMessage continuedAsNewMessage, - OrchestrationState orchestrationState) - { - this.instanceStore.SaveState( - runtimeState: newOrchestrationRuntimeState, - statusRecord: orchestrationState, - newMessages: orchestratorMessages.Union(timerMessages).Append(continuedAsNewMessage).Where(msg => msg != null)); - - this.activityQueue.Enqueue(outboundMessages); - return Task.CompletedTask; - } - - public Task CreateAsync() => Task.CompletedTask; - - public Task CreateAsync(bool recreateInstanceStore) - { - if (recreateInstanceStore) - { - this.instanceStore.Reset(); - } - return Task.CompletedTask; - } - - public Task CreateIfNotExistsAsync() => Task.CompletedTask; - - public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage) - { - return this.CreateTaskOrchestrationAsync( - creationMessage, - new[] { OrchestrationStatus.Pending, OrchestrationStatus.Running }); - } - - public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[]? dedupeStatuses) - { - // Lock the instance store to prevent multiple "create" threads from racing with each other. - lock (this.instanceStore) - { - string instanceId = creationMessage.OrchestrationInstance.InstanceId; - if (this.instanceStore.TryGetState(instanceId, out OrchestrationState? statusRecord) && - dedupeStatuses != null && - dedupeStatuses.Contains(statusRecord.OrchestrationStatus)) - { - throw new OrchestrationAlreadyExistsException($"An orchestration with id '{instanceId}' already exists. It's in the {statusRecord.OrchestrationStatus} state."); - } - - this.instanceStore.AddMessage(creationMessage); - } - - return Task.CompletedTask; - } - - public Task DeleteAsync() => this.DeleteAsync(true); - - public Task DeleteAsync(bool deleteInstanceStore) - { - if (deleteInstanceStore) - { - this.instanceStore.Reset(); - } - return Task.CompletedTask; - } - - public Task ForceTerminateTaskOrchestrationAsync(string instanceId, string reason) - { - var taskMessage = new TaskMessage - { - OrchestrationInstance = new OrchestrationInstance { InstanceId = instanceId }, - Event = new ExecutionTerminatedEvent(-1, reason), - }; - - return this.SendTaskOrchestrationMessageAsync(taskMessage); - } - - public int GetDelayInSecondsAfterOnFetchException(Exception exception) - { - return exception is OperationCanceledException ? 0 : 1; - } - - public int GetDelayInSecondsAfterOnProcessException(Exception exception) - { - return exception is OperationCanceledException ? 0 : 1; - } - - public Task GetOrchestrationHistoryAsync(string instanceId, string executionId) - { - // Also not supported in the emulator - throw new NotImplementedException(); - } - - public async Task> GetOrchestrationStateAsync(string instanceId, bool allExecutions) - { - OrchestrationState state = await this.GetOrchestrationStateAsync(instanceId, executionId: null); - return new[] { state }; - } - - public Task GetOrchestrationStateAsync(string instanceId, string? executionId) - { - if (this.instanceStore.TryGetState(instanceId, out OrchestrationState? statusRecord)) - { - if (executionId == null || executionId == statusRecord.OrchestrationInstance.ExecutionId) - { - return Task.FromResult(statusRecord); - } - } - - return Task.FromResult(null!); - } - - public bool IsMaxMessageCountExceeded(int currentMessageCount, OrchestrationRuntimeState runtimeState) => false; - - public async Task LockNextTaskActivityWorkItem(TimeSpan receiveTimeout, CancellationToken cancellationToken) - { - TaskMessage message = await this.activityQueue.DequeueAsync(cancellationToken); - return new TaskActivityWorkItem - { - Id = message.SequenceNumber.ToString(), - LockedUntilUtc = DateTime.MaxValue, - TaskMessage = message, - }; - } - - public async Task LockNextTaskOrchestrationWorkItemAsync(TimeSpan receiveTimeout, CancellationToken cancellationToken) - { - var (instanceId, runtimeState, messages) = await this.instanceStore.GetNextReadyToRunInstanceAsync(cancellationToken); - - return new TaskOrchestrationWorkItem - { - InstanceId = instanceId, - OrchestrationRuntimeState = runtimeState, - NewMessages = messages, - LockedUntilUtc = DateTime.MaxValue, - }; - } - - public Task PurgeOrchestrationHistoryAsync(DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType) - { - // Also not supported in the emulator - throw new NotImplementedException(); - } - - public Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) - { - this.instanceStore.ReleaseLock(workItem.InstanceId); - return Task.CompletedTask; - } - - public Task RenewTaskActivityWorkItemLockAsync(TaskActivityWorkItem workItem) - { - return Task.FromResult(workItem); // PeekLock isn't supported - } - - public Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem) - { - return Task.CompletedTask; // PeekLock isn't supported - } - - public Task SendTaskOrchestrationMessageAsync(TaskMessage message) - { - this.instanceStore.AddMessage(message); - return Task.CompletedTask; - } - - public Task SendTaskOrchestrationMessageBatchAsync(params TaskMessage[] messages) - { - // NOTE: This is not transactionally consistent - some messages may get processed earlier than others. - foreach (TaskMessage message in messages) - { - this.instanceStore.AddMessage(message); - } - - return Task.CompletedTask; - } - - public Task StartAsync() => Task.CompletedTask; - - public Task StopAsync() => Task.CompletedTask; - - public Task StopAsync(bool isForced) => Task.CompletedTask; - - public async Task WaitForOrchestrationAsync(string instanceId, string executionId, TimeSpan timeout, CancellationToken cancellationToken) - { - if (timeout <= TimeSpan.Zero) - { - return await this.instanceStore.WaitForInstanceAsync(instanceId, cancellationToken); - } - else - { - using CancellationTokenSource timeoutCancellationSource = new(timeout); - using CancellationTokenSource linkedCancellationSource = CancellationTokenSource.CreateLinkedTokenSource( - cancellationToken, - timeoutCancellationSource.Token); - return await this.instanceStore.WaitForInstanceAsync(instanceId, linkedCancellationSource.Token); - } - } - - static bool TryGetScheduledTime(TaskMessage message, out TimeSpan delay) - { - DateTime scheduledTime = default; - if (message.Event is ExecutionStartedEvent startEvent) - { - scheduledTime = startEvent.ScheduledStartTime ?? default; - } - else if (message.Event is TimerFiredEvent timerEvent) - { - scheduledTime = timerEvent.FireAt; - } - - DateTime now = DateTime.UtcNow; - if (scheduledTime > now) - { - delay = scheduledTime - now; - return true; - } - else - { - delay = default; - return false; - } - } - - public Task GetOrchestrationWithQueryAsync(OrchestrationQuery query, CancellationToken cancellationToken) - { - return Task.FromResult(this.instanceStore.GetOrchestrationWithQuery(query)); - } - - public Task PurgeInstanceStateAsync(string instanceId) - { - return Task.FromResult(this.instanceStore.PurgeInstanceState(instanceId)); - } - - public Task PurgeInstanceStateAsync(PurgeInstanceFilter purgeInstanceFilter) - { - return Task.FromResult(this.instanceStore.PurgeInstanceState(purgeInstanceFilter)); - } - - class InMemoryQueue - { - readonly Channel innerQueue = Channel.CreateUnbounded(); - - public void Enqueue(TaskMessage taskMessage) - { - if (TryGetScheduledTime(taskMessage, out TimeSpan delay)) - { - _ = Task.Delay(delay).ContinueWith(t => this.innerQueue.Writer.TryWrite(taskMessage)); - } - else - { - this.innerQueue.Writer.TryWrite(taskMessage); - } - } - - public void Enqueue(IEnumerable messages) - { - foreach (TaskMessage msg in messages) - { - this.Enqueue(msg); - } - } - - public async Task DequeueAsync(CancellationToken cancellationToken) - { - return await this.innerQueue.Reader.ReadAsync(cancellationToken); - } - } - - class InMemoryInstanceStore - { - readonly ConcurrentDictionary store = new(StringComparer.OrdinalIgnoreCase); - readonly ConcurrentDictionary> waiters = new(StringComparer.OrdinalIgnoreCase); - readonly ReadyToRunQueue readyToRunQueue = new(); - - readonly ILogger logger; - - public InMemoryInstanceStore(ILogger logger) => this.logger = logger; - - public void Reset() - { - this.store.Clear(); - this.waiters.Clear(); - this.readyToRunQueue.Reset(); - } - - public async Task<(string, OrchestrationRuntimeState, List)> GetNextReadyToRunInstanceAsync(CancellationToken cancellationToken) - { - SerializedInstanceState state = await this.readyToRunQueue.TakeNextAsync(cancellationToken); - lock (state) - { - List history = state.HistoryEventsJson.Select(e => e!.GetValue()).ToList(); - OrchestrationRuntimeState runtimeState = new(history); - - List messages = state.MessagesJson.Select(node => node!.GetValue()).ToList(); - if (messages == null) - { - throw new InvalidOperationException("Should never load state with zero messages."); - } - - state.IsLoaded = true; - - // There is no "peek-lock" semantic. All dequeued messages are immediately deleted. - state.MessagesJson.Clear(); - - return (state.InstanceId, runtimeState, messages); - } - } - - public bool TryGetState(string instanceId, [NotNullWhen(true)] out OrchestrationState? statusRecord) - { - if (!this.store.TryGetValue(instanceId, out SerializedInstanceState? state)) - { - statusRecord = null; - return false; - } - - statusRecord = state.StatusRecordJson?.GetValue(); - return statusRecord != null; - } - - public void SaveState( - OrchestrationRuntimeState runtimeState, - OrchestrationState statusRecord, - IEnumerable newMessages) - { - static bool IsCompleted(OrchestrationRuntimeState runtimeState) => - runtimeState.OrchestrationStatus == OrchestrationStatus.Completed || - runtimeState.OrchestrationStatus == OrchestrationStatus.Failed || - runtimeState.OrchestrationStatus == OrchestrationStatus.Terminated || - runtimeState.OrchestrationStatus == OrchestrationStatus.Canceled; - - if (string.IsNullOrEmpty(runtimeState.OrchestrationInstance?.InstanceId)) - { - throw new ArgumentException("The provided runtime state doesn't contain instance ID information.", nameof(runtimeState)); - } - - string instanceId = runtimeState.OrchestrationInstance.InstanceId; - string executionId = runtimeState.OrchestrationInstance.ExecutionId; - SerializedInstanceState state = this.store.GetOrAdd( - instanceId, - _ => new SerializedInstanceState(instanceId, executionId)); - lock (state) - { - if (state.ExecutionId == null) - { - // This orchestration was started by a message without an execution ID. - state.ExecutionId = executionId; - } - else if (state.ExecutionId != executionId) - { - // This is a new generation (ContinueAsNew). Erase the old history. - state.HistoryEventsJson.Clear(); - state.ExecutionId = executionId; - } - - foreach (TaskMessage msg in newMessages) - { - this.AddMessage(msg); - } - - // Append to the orchestration history - foreach (HistoryEvent e in runtimeState.NewEvents) - { - state.HistoryEventsJson.Add(e); - } - - state.StatusRecordJson = JsonValue.Create(statusRecord); - state.IsCompleted = IsCompleted(runtimeState); - } - - // Notify any waiters of the orchestration completion - if (IsCompleted(runtimeState) && - this.waiters.TryRemove(statusRecord.OrchestrationInstance.InstanceId, out TaskCompletionSource? waiter)) - { - waiter.TrySetResult(statusRecord); - } - } - - public void AddMessage(TaskMessage message) - { - string instanceId = message.OrchestrationInstance.InstanceId; - string? executionId = message.OrchestrationInstance.ExecutionId; - - SerializedInstanceState state = this.store.GetOrAdd(instanceId, id => new SerializedInstanceState(id, executionId)); - lock (state) - { - bool isRestart = state.ExecutionId != null && state.ExecutionId != executionId; - - if (message.Event is ExecutionStartedEvent startEvent) - { - // For restart scenarios, clear the history and reset the state - if (isRestart && state.IsCompleted) - { - state.ExecutionId = executionId; - state.IsLoaded = false; - } - - OrchestrationState newStatusRecord = new() - { - OrchestrationInstance = startEvent.OrchestrationInstance, - CreatedTime = DateTime.UtcNow, - LastUpdatedTime = DateTime.UtcNow, - OrchestrationStatus = OrchestrationStatus.Pending, - Version = startEvent.Version, - Name = startEvent.Name, - Input = startEvent.Input, - ScheduledStartTime = startEvent.ScheduledStartTime, - Tags = startEvent.Tags, - }; - - state.StatusRecordJson = JsonValue.Create(newStatusRecord); - state.HistoryEventsJson.Clear(); - state.IsCompleted = false; - } - else if (state.IsCompleted) - { - // Drop the message since we're completed - // GOOD: The user-provided the instanceId - // logger.LogWarning( - // "Dropped {eventType} message for instance '{instanceId}' because the orchestration has already completed.", - // message.Event.EventType, - // instanceId); - return; - } - - if (TryGetScheduledTime(message, out TimeSpan delay)) - { - // Not ready for this message yet - delay the enqueue - _ = Task.Delay(delay).ContinueWith(t => this.AddMessage(message)); - return; - } - - state.MessagesJson.Add(message); - - if (!state.IsLoaded) - { - // The orchestration isn't running, so schedule it to run now. - // If it is running, it will be scheduled again automatically when it's released. - this.readyToRunQueue.Schedule(state); - } - } - } - - public void AbandonInstance(IEnumerable messagesToReturn) - { - foreach (var message in messagesToReturn) - { - this.AddMessage(message); - } - } - - public void ReleaseLock(string instanceId) - { - if (!this.store.TryGetValue(instanceId, out SerializedInstanceState? state) || !state.IsLoaded) - { - throw new InvalidOperationException($"Instance {instanceId} is not in the store or is not loaded!"); - } - - lock (state) - { - state.IsLoaded = false; - if (state.MessagesJson.Count > 0) - { - // More messages came in while we were running. Or, messages were abandoned. - // Put this back into the read-to-run queue! - this.readyToRunQueue.Schedule(state); - } - } - } - - public Task WaitForInstanceAsync(string instanceId, CancellationToken cancellationToken) - { - if (this.store.TryGetValue(instanceId, out SerializedInstanceState? state)) - { - lock (state) - { - OrchestrationState? statusRecord = state.StatusRecordJson?.GetValue(); - if (statusRecord != null) - { - if (statusRecord.OrchestrationStatus == OrchestrationStatus.Completed || - statusRecord.OrchestrationStatus == OrchestrationStatus.Failed || - statusRecord.OrchestrationStatus == OrchestrationStatus.Terminated) - { - // orchestration has already completed - return Task.FromResult(statusRecord); - } - } - - } - } - - // Caller will be notified when the instance completes. - // The ContinueWith is just to enable cancellation: https://stackoverflow.com/a/25652873/2069 - var tcs = this.waiters.GetOrAdd(instanceId, _ => new TaskCompletionSource()); - return tcs.Task.ContinueWith(t => t.GetAwaiter().GetResult(), cancellationToken); - } - - public OrchestrationQueryResult GetOrchestrationWithQuery(OrchestrationQuery query) - { - int startIndex = 0; - int counter = 0; - string? continuationToken = query.ContinuationToken; - if (continuationToken != null) - { - if (!Int32.TryParse(continuationToken, out startIndex)) - { - throw new InvalidOperationException($"{continuationToken} cannot be parsed to Int32"); - } - } - - counter = startIndex; - - List results = this.store - .Skip(startIndex) - .Where(item => - { - counter++; - OrchestrationState? statusRecord = item.Value.StatusRecordJson?.GetValue(); - if (statusRecord == null) return false; - if (query.CreatedTimeFrom != null && query.CreatedTimeFrom > statusRecord.CreatedTime) return false; - if (query.CreatedTimeTo != null && query.CreatedTimeTo < statusRecord.CreatedTime) return false; - if (query.RuntimeStatus != null && query.RuntimeStatus.Any() && !query.RuntimeStatus.Contains(statusRecord.OrchestrationStatus)) return false; - if (query.InstanceIdPrefix != null && !statusRecord.OrchestrationInstance.InstanceId.StartsWith(query.InstanceIdPrefix)) return false; - return true; - }) - .Take(query.PageSize) - .Select(item => item.Value.StatusRecordJson!.GetValue()) - .ToList(); - - string? token = null; - if (results.Count == query.PageSize) - { - token = counter.ToString(); - } - return new OrchestrationQueryResult(results, token); - } - - public PurgeResult PurgeInstanceState(string instanceId) - { - if (instanceId != null && this.store.TryGetValue(instanceId, out SerializedInstanceState? state) && state.IsCompleted) - { - this.store.TryRemove(instanceId, out SerializedInstanceState? removedState); - if (removedState != null) - { - return new PurgeResult(1); - } - } - return new PurgeResult(0); - } - - public PurgeResult PurgeInstanceState(PurgeInstanceFilter purgeInstanceFilter) - { - int counter = 0; - - List filteredInstanceIds = this.store - .Where(item => - { - OrchestrationState? statusRecord = item.Value.StatusRecordJson?.GetValue(); - if (statusRecord == null) return false; - if (purgeInstanceFilter.CreatedTimeFrom > statusRecord.CreatedTime) return false; - if (purgeInstanceFilter.CreatedTimeTo != null && purgeInstanceFilter.CreatedTimeTo < statusRecord.CreatedTime) return false; - if (purgeInstanceFilter.RuntimeStatus != null && purgeInstanceFilter.RuntimeStatus.Any() && !purgeInstanceFilter.RuntimeStatus.Contains(statusRecord.OrchestrationStatus)) return false; - return true; - }) - .Select(item => item.Key) - .ToList(); - - foreach (string instanceId in filteredInstanceIds) - { - this.store.TryRemove(instanceId, out SerializedInstanceState? removedState); - if (removedState != null) - { - counter++; - } - } - - return new PurgeResult(counter); - } - - class ReadyToRunQueue - { - readonly Channel readyToRunQueue = Channel.CreateUnbounded(); - readonly Dictionary readyInstances = new(StringComparer.OrdinalIgnoreCase); - - public void Reset() - { - this.readyInstances.Clear(); - } - - public async ValueTask TakeNextAsync(CancellationToken ct) - { - while (true) - { - SerializedInstanceState state = await this.readyToRunQueue.Reader.ReadAsync(ct); - lock (state) - { - if (this.readyInstances.Remove(state.InstanceId)) - { - if (state.IsLoaded) - { - throw new InvalidOperationException("Should never load state that is already loaded."); - } - - state.IsLoaded = true; - return state; - } - } - } - } - - public void Schedule(SerializedInstanceState state) - { - // TODO: There is a race condition here. If another thread is calling TakeNextAsync - // and removed the queue item before updating the dictionary, then we'll fail - // to update the readyToRunQueue and the orchestration will get stuck. - if (this.readyInstances.TryAdd(state.InstanceId, state)) - { - if (!this.readyToRunQueue.Writer.TryWrite(state)) - { - throw new InvalidOperationException($"unable to write to queue for {state.InstanceId}"); - } - } - } - } - - class SerializedInstanceState - { - public SerializedInstanceState(string instanceId, string? executionId) - { - this.InstanceId = instanceId; - this.ExecutionId = executionId; - } - - public string InstanceId { get; } - public string? ExecutionId { get; internal set; } - public JsonValue? StatusRecordJson { get; set; } - public JsonArray HistoryEventsJson { get; } = new JsonArray(); - public JsonArray MessagesJson { get; } = new JsonArray(); - - internal bool IsLoaded { get; set; } - internal bool IsCompleted { get; set; } - } - } -} +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Collections.Concurrent; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Text.Json.Nodes; +using System.Threading.Channels; +using DurableTask.Core; +using DurableTask.Core.Exceptions; +using DurableTask.Core.History; +using DurableTask.Core.Query; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Microsoft.DurableTask.Testing.Sidecar; + +/// +/// An in-memory implementation of orchestration service interfaces for testing purposes. +/// +/// +/// This class provides a complete in-memory implementation of orchestration services, +/// allowing for testing of durable task orchestrations without external dependencies. +/// +public class InMemoryOrchestrationService : IOrchestrationService, IOrchestrationServiceClient, IOrchestrationServiceQueryClient, IOrchestrationServicePurgeClient +{ + readonly InMemoryQueue activityQueue = new(); + readonly InMemoryInstanceStore instanceStore; + readonly ILogger logger; + + /// + /// Gets the number of task orchestration dispatchers. + /// + public int TaskOrchestrationDispatcherCount => 1; + + /// + /// Gets the number of task activity dispatchers. + /// + public int TaskActivityDispatcherCount => 1; + + /// + /// Gets the maximum number of concurrent task orchestration work items. + /// + public int MaxConcurrentTaskOrchestrationWorkItems => Environment.ProcessorCount; + + /// + /// Gets the maximum number of concurrent task activity work items. + /// + public int MaxConcurrentTaskActivityWorkItems => Environment.ProcessorCount; + + /// + /// Gets the behavior for events when continuing as new. + /// + public BehaviorOnContinueAsNew EventBehaviourForContinueAsNew => BehaviorOnContinueAsNew.Carryover; + + /// + /// Initializes a new instance of the class. + /// + /// Optional logger factory for creating loggers. + public InMemoryOrchestrationService(ILoggerFactory? loggerFactory = null) + { + this.logger = (loggerFactory ?? NullLoggerFactory.Instance).CreateLogger("Microsoft.DurableTask.Sidecar.InMemoryStorageProvider"); + this.instanceStore = new InMemoryInstanceStore(this.logger); + } + + /// + /// Abandons a task activity work item. + /// + /// The work item to abandon. + /// A task representing the asynchronous operation. + public Task AbandonTaskActivityWorkItemAsync(TaskActivityWorkItem workItem) + { + this.logger.LogWarning("Abandoning task activity work item {Id}", workItem.Id); + this.activityQueue.Enqueue(workItem.TaskMessage); + return Task.CompletedTask; + } + + /// + /// Abandons a task orchestration work item. + /// + /// The work item to abandon. + /// A task representing the asynchronous operation. + public Task AbandonTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) + { + this.instanceStore.AbandonInstance(workItem.NewMessages); + return Task.CompletedTask; + } + + /// + /// Completes a task activity work item. + /// + /// The work item to complete. + /// The response message. + /// A task representing the asynchronous operation. + public Task CompleteTaskActivityWorkItemAsync(TaskActivityWorkItem workItem, TaskMessage responseMessage) + { + this.instanceStore.AddMessage(responseMessage); + return Task.CompletedTask; + } + + /// + /// Completes a task orchestration work item. + /// + /// The work item to complete. + /// The new orchestration runtime state. + /// The outbound messages. + /// The orchestrator messages. + /// The timer messages. + /// The continued as new message. + /// The orchestration state. + /// A task representing the asynchronous operation. + public Task CompleteTaskOrchestrationWorkItemAsync( + TaskOrchestrationWorkItem workItem, + OrchestrationRuntimeState newOrchestrationRuntimeState, + IList outboundMessages, + IList orchestratorMessages, + IList timerMessages, + TaskMessage continuedAsNewMessage, + OrchestrationState orchestrationState) + { + this.instanceStore.SaveState( + runtimeState: newOrchestrationRuntimeState, + statusRecord: orchestrationState, + newMessages: orchestratorMessages.Union(timerMessages).Append(continuedAsNewMessage).Where(msg => msg != null)); + + this.activityQueue.Enqueue(outboundMessages); + return Task.CompletedTask; + } + + /// + /// Creates the orchestration service. + /// + /// A task representing the asynchronous operation. + public Task CreateAsync() => Task.CompletedTask; + + /// + /// Creates the orchestration service. + /// + /// Whether to recreate the instance store. + /// A task representing the asynchronous operation. + public Task CreateAsync(bool recreateInstanceStore) + { + if (recreateInstanceStore) + { + this.instanceStore.Reset(); + } + return Task.CompletedTask; + } + + /// + /// Creates the orchestration service if it does not exist. + /// + /// A task representing the asynchronous operation. + public Task CreateIfNotExistsAsync() => Task.CompletedTask; + + /// + /// Creates a task orchestration. + /// + /// The creation message. + /// A task representing the asynchronous operation. + public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage) + { + return this.CreateTaskOrchestrationAsync( + creationMessage, + new[] { OrchestrationStatus.Pending, OrchestrationStatus.Running }); + } + + /// + /// Creates a task orchestration. + /// + /// The creation message. + /// The deduplication statuses. + /// A task representing the asynchronous operation. + public Task CreateTaskOrchestrationAsync(TaskMessage creationMessage, OrchestrationStatus[]? dedupeStatuses) + { + // Lock the instance store to prevent multiple "create" threads from racing with each other. + lock (this.instanceStore) + { + string instanceId = creationMessage.OrchestrationInstance.InstanceId; + if (this.instanceStore.TryGetState(instanceId, out OrchestrationState? statusRecord) && + dedupeStatuses != null && + dedupeStatuses.Contains(statusRecord.OrchestrationStatus)) + { + throw new OrchestrationAlreadyExistsException($"An orchestration with id '{instanceId}' already exists. It's in the {statusRecord.OrchestrationStatus} state."); + } + + this.instanceStore.AddMessage(creationMessage); + } + + return Task.CompletedTask; + } + + /// + /// Deletes the orchestration service. + /// + /// A task representing the asynchronous operation. + public Task DeleteAsync() => this.DeleteAsync(true); + + /// + /// Deletes the orchestration service. + /// + /// Whether to delete the instance store. + /// A task representing the asynchronous operation. + public Task DeleteAsync(bool deleteInstanceStore) + { + if (deleteInstanceStore) + { + this.instanceStore.Reset(); + } + return Task.CompletedTask; + } + + /// + /// Force terminates a task orchestration. + /// + /// The instance ID. + /// The termination reason. + /// A task representing the asynchronous operation. + public Task ForceTerminateTaskOrchestrationAsync(string instanceId, string reason) + { + var taskMessage = new TaskMessage + { + OrchestrationInstance = new OrchestrationInstance { InstanceId = instanceId }, + Event = new ExecutionTerminatedEvent(-1, reason), + }; + + return this.SendTaskOrchestrationMessageAsync(taskMessage); + } + + /// + /// Gets the delay in seconds after an on-fetch exception. + /// + /// The exception that occurred. + /// The delay in seconds. + public int GetDelayInSecondsAfterOnFetchException(Exception exception) + { + return exception is OperationCanceledException ? 0 : 1; + } + + /// + /// Gets the delay in seconds after an on-process exception. + /// + /// The exception that occurred. + /// The delay in seconds. + public int GetDelayInSecondsAfterOnProcessException(Exception exception) + { + return exception is OperationCanceledException ? 0 : 1; + } + + /// + /// Gets the orchestration history. + /// + /// The instance ID. + /// The execution ID. + /// A task representing the asynchronous operation. + /// This method is not implemented in the in-memory service. + public Task GetOrchestrationHistoryAsync(string instanceId, string executionId) + { + // Also not supported in the emulator + throw new NotImplementedException(); + } + + /// + /// Gets the orchestration state. + /// + /// The instance ID. + /// Whether to get all executions. + /// A task representing the asynchronous operation. + public async Task> GetOrchestrationStateAsync(string instanceId, bool allExecutions) + { + OrchestrationState state = await this.GetOrchestrationStateAsync(instanceId, executionId: null); + return new[] { state }; + } + + /// + /// Gets the orchestration state. + /// + /// The instance ID. + /// The execution ID. + /// A task representing the asynchronous operation. + public Task GetOrchestrationStateAsync(string instanceId, string? executionId) + { + if (this.instanceStore.TryGetState(instanceId, out OrchestrationState? statusRecord)) + { + if (executionId == null || executionId == statusRecord.OrchestrationInstance.ExecutionId) + { + return Task.FromResult(statusRecord); + } + } + + return Task.FromResult(null!); + } + + /// + /// Determines if the maximum message count is exceeded. + /// + /// The current message count. + /// The runtime state. + /// Always returns false for the in-memory service. + public bool IsMaxMessageCountExceeded(int currentMessageCount, OrchestrationRuntimeState runtimeState) => false; + + /// + /// Locks the next task activity work item. + /// + /// The receive timeout. + /// The cancellation token. + /// A task representing the asynchronous operation. + public async Task LockNextTaskActivityWorkItem(TimeSpan receiveTimeout, CancellationToken cancellationToken) + { + TaskMessage message = await this.activityQueue.DequeueAsync(cancellationToken); + return new TaskActivityWorkItem + { + Id = message.SequenceNumber.ToString(CultureInfo.InvariantCulture), // CA1305: Use IFormatProvider + LockedUntilUtc = DateTime.MaxValue, + TaskMessage = message, + }; + } + + /// + /// Locks the next task orchestration work item. + /// + /// The receive timeout. + /// The cancellation token. + /// A task representing the asynchronous operation. + public async Task LockNextTaskOrchestrationWorkItemAsync(TimeSpan receiveTimeout, CancellationToken cancellationToken) + { + var (instanceId, runtimeState, messages) = await this.instanceStore.GetNextReadyToRunInstanceAsync(cancellationToken); + + return new TaskOrchestrationWorkItem + { + InstanceId = instanceId, + OrchestrationRuntimeState = runtimeState, + NewMessages = messages, + LockedUntilUtc = DateTime.MaxValue, + }; + } + + /// + /// Purges orchestration history. + /// + /// The threshold date time in UTC. + /// The time range filter type. + /// A task representing the asynchronous operation. + /// This method is not implemented in the in-memory service. + public Task PurgeOrchestrationHistoryAsync(DateTime thresholdDateTimeUtc, OrchestrationStateTimeRangeFilterType timeRangeFilterType) + { + // Also not supported in the emulator + throw new NotImplementedException(); + } + + /// + /// Releases a task orchestration work item. + /// + /// The work item to release. + /// A task representing the asynchronous operation. + public Task ReleaseTaskOrchestrationWorkItemAsync(TaskOrchestrationWorkItem workItem) + { + this.instanceStore.ReleaseLock(workItem.InstanceId); + return Task.CompletedTask; + } + + /// + /// Renews a task activity work item lock. + /// + /// The work item to renew. + /// A task representing the asynchronous operation. + public Task RenewTaskActivityWorkItemLockAsync(TaskActivityWorkItem workItem) + { + return Task.FromResult(workItem); // PeekLock isn't supported + } + + /// + /// Renews a task orchestration work item lock. + /// + /// The work item to renew. + /// A task representing the asynchronous operation. + public Task RenewTaskOrchestrationWorkItemLockAsync(TaskOrchestrationWorkItem workItem) + { + return Task.CompletedTask; // PeekLock isn't supported + } + + /// + /// Sends a task orchestration message. + /// + /// The message to send. + /// A task representing the asynchronous operation. + public Task SendTaskOrchestrationMessageAsync(TaskMessage message) + { + this.instanceStore.AddMessage(message); + return Task.CompletedTask; + } + + /// + /// Sends a batch of task orchestration messages. + /// + /// The messages to send. + /// A task representing the asynchronous operation. + public Task SendTaskOrchestrationMessageBatchAsync(params TaskMessage[] messages) + { + // NOTE: This is not transactionally consistent - some messages may get processed earlier than others. + foreach (TaskMessage message in messages) + { + this.instanceStore.AddMessage(message); + } + + return Task.CompletedTask; + } + + /// + /// Starts the orchestration service. + /// + /// A task representing the asynchronous operation. + public Task StartAsync() => Task.CompletedTask; + + /// + /// Stops the orchestration service. + /// + /// A task representing the asynchronous operation. + public Task StopAsync() => Task.CompletedTask; + + /// + /// Stops the orchestration service. + /// + /// Whether the stop is forced. + /// A task representing the asynchronous operation. + public Task StopAsync(bool isForced) => Task.CompletedTask; + + /// + /// Waits for an orchestration to complete. + /// + /// The instance ID. + /// The execution ID. + /// The timeout duration. + /// The cancellation token. + /// A task representing the asynchronous operation. + public async Task WaitForOrchestrationAsync(string instanceId, string executionId, TimeSpan timeout, CancellationToken cancellationToken) + { + if (timeout <= TimeSpan.Zero) + { + return await this.instanceStore.WaitForInstanceAsync(instanceId, cancellationToken); + } + else + { + using CancellationTokenSource timeoutCancellationSource = new(timeout); + using CancellationTokenSource linkedCancellationSource = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + timeoutCancellationSource.Token); + return await this.instanceStore.WaitForInstanceAsync(instanceId, linkedCancellationSource.Token); + } + } + + static bool TryGetScheduledTime(TaskMessage message, out TimeSpan delay) + { + DateTime scheduledTime = default; + if (message.Event is ExecutionStartedEvent startEvent) + { + scheduledTime = startEvent.ScheduledStartTime ?? default; + } + else if (message.Event is TimerFiredEvent timerEvent) + { + scheduledTime = timerEvent.FireAt; + } + + DateTime now = DateTime.UtcNow; + if (scheduledTime > now) + { + delay = scheduledTime - now; + return true; + } + else + { + delay = default; + return false; + } + } + + /// + /// Gets orchestrations with a query. + /// + /// The orchestration query. + /// The cancellation token. + /// A task representing the asynchronous operation. + public Task GetOrchestrationWithQueryAsync(OrchestrationQuery query, CancellationToken cancellationToken) + { + return Task.FromResult(this.instanceStore.GetOrchestrationWithQuery(query)); + } + + /// + /// Purges the state of a specific orchestration instance. + /// + /// The instance ID to purge. + /// A task representing the asynchronous operation. + public Task PurgeInstanceStateAsync(string instanceId) + { + return Task.FromResult(this.instanceStore.PurgeInstanceState(instanceId)); + } + + /// + /// Purges orchestration instances based on a filter. + /// + /// The purge instance filter. + /// A task representing the asynchronous operation. + public Task PurgeInstanceStateAsync(PurgeInstanceFilter purgeInstanceFilter) + { + return Task.FromResult(this.instanceStore.PurgeInstanceState(purgeInstanceFilter)); + } + + class InMemoryQueue + { + readonly Channel innerQueue = Channel.CreateUnbounded(); + + public void Enqueue(TaskMessage taskMessage) + { + if (TryGetScheduledTime(taskMessage, out TimeSpan delay)) + { + _ = Task.Delay(delay).ContinueWith(t => this.innerQueue.Writer.TryWrite(taskMessage)); + } + else + { + this.innerQueue.Writer.TryWrite(taskMessage); + } + } + + public void Enqueue(IEnumerable messages) + { + foreach (TaskMessage msg in messages) + { + this.Enqueue(msg); + } + } + + public async Task DequeueAsync(CancellationToken cancellationToken) + { + return await this.innerQueue.Reader.ReadAsync(cancellationToken); + } + } + + class InMemoryInstanceStore + { + readonly ConcurrentDictionary store = new(StringComparer.OrdinalIgnoreCase); + readonly ConcurrentDictionary> waiters = new(StringComparer.OrdinalIgnoreCase); + readonly ReadyToRunQueue readyToRunQueue = new(); + + readonly ILogger logger; + + public InMemoryInstanceStore(ILogger logger) => this.logger = logger; + + public void Reset() + { + this.store.Clear(); + this.waiters.Clear(); + this.readyToRunQueue.Reset(); + } + + public async Task<(string, OrchestrationRuntimeState, List)> GetNextReadyToRunInstanceAsync(CancellationToken cancellationToken) + { + SerializedInstanceState state = await this.readyToRunQueue.TakeNextAsync(cancellationToken); + lock (state) + { + List history = state.HistoryEventsJson.Select(e => e!.GetValue()).ToList(); + OrchestrationRuntimeState runtimeState = new(history); + + List messages = state.MessagesJson.Select(node => node!.GetValue()).ToList(); + if (messages == null) + { + throw new InvalidOperationException("Should never load state with zero messages."); + } + + state.IsLoaded = true; + + // There is no "peek-lock" semantic. All dequeued messages are immediately deleted. + state.MessagesJson.Clear(); + + return (state.InstanceId, runtimeState, messages); + } + } + + public bool TryGetState(string instanceId, [NotNullWhen(true)] out OrchestrationState? statusRecord) + { + if (!this.store.TryGetValue(instanceId, out SerializedInstanceState? state)) + { + statusRecord = null; + return false; + } + + statusRecord = state.StatusRecordJson?.GetValue(); + return statusRecord != null; + } + + public void SaveState( + OrchestrationRuntimeState runtimeState, + OrchestrationState statusRecord, + IEnumerable newMessages) + { + static bool IsCompleted(OrchestrationRuntimeState runtimeState) => + runtimeState.OrchestrationStatus == OrchestrationStatus.Completed || + runtimeState.OrchestrationStatus == OrchestrationStatus.Failed || + runtimeState.OrchestrationStatus == OrchestrationStatus.Terminated || + runtimeState.OrchestrationStatus == OrchestrationStatus.Canceled; + + if (string.IsNullOrEmpty(runtimeState.OrchestrationInstance?.InstanceId)) + { + throw new ArgumentException("The provided runtime state doesn't contain instance ID information.", nameof(runtimeState)); + } + + string instanceId = runtimeState.OrchestrationInstance.InstanceId; + string executionId = runtimeState.OrchestrationInstance.ExecutionId; + SerializedInstanceState state = this.store.GetOrAdd( + instanceId, + _ => new SerializedInstanceState(instanceId, executionId)); + lock (state) + { + if (state.ExecutionId == null) + { + // This orchestration was started by a message without an execution ID. + state.ExecutionId = executionId; + } + else if (state.ExecutionId != executionId) + { + // This is a new generation (ContinueAsNew). Erase the old history. + state.HistoryEventsJson.Clear(); + state.ExecutionId = executionId; + } + + foreach (TaskMessage msg in newMessages) + { + this.AddMessage(msg); + } + + // Append to the orchestration history + foreach (HistoryEvent e in runtimeState.NewEvents) + { + state.HistoryEventsJson.Add(e); + } + + state.StatusRecordJson = JsonValue.Create(statusRecord); + state.IsCompleted = IsCompleted(runtimeState); + } + + // Notify any waiters of the orchestration completion + if (IsCompleted(runtimeState) && + this.waiters.TryRemove(statusRecord.OrchestrationInstance.InstanceId, out TaskCompletionSource? waiter)) + { + waiter.TrySetResult(statusRecord); + } + } + + public void AddMessage(TaskMessage message) + { + string instanceId = message.OrchestrationInstance.InstanceId; + string? executionId = message.OrchestrationInstance.ExecutionId; + + SerializedInstanceState state = this.store.GetOrAdd(instanceId, id => new SerializedInstanceState(id, executionId)); + lock (state) + { + bool isRestart = state.ExecutionId != null && state.ExecutionId != executionId; + + if (message.Event is ExecutionStartedEvent startEvent) + { + // For restart scenarios, clear the history and reset the state + if (isRestart && state.IsCompleted) + { + state.ExecutionId = executionId; + state.IsLoaded = false; + } + + OrchestrationState newStatusRecord = new() + { + OrchestrationInstance = startEvent.OrchestrationInstance, + CreatedTime = DateTime.UtcNow, + LastUpdatedTime = DateTime.UtcNow, + OrchestrationStatus = OrchestrationStatus.Pending, + Version = startEvent.Version, + Name = startEvent.Name, + Input = startEvent.Input, + ScheduledStartTime = startEvent.ScheduledStartTime, + Tags = startEvent.Tags, + }; + + state.StatusRecordJson = JsonValue.Create(newStatusRecord); + state.HistoryEventsJson.Clear(); + state.IsCompleted = false; + } + else if (state.IsCompleted) + { + // Drop the message since we're completed + // GOOD: The user-provided the instanceId + // logger.LogWarning( + // "Dropped {eventType} message for instance '{instanceId}' because the orchestration has already completed.", + // message.Event.EventType, + // instanceId); + return; + } + + if (TryGetScheduledTime(message, out TimeSpan delay)) + { + // Not ready for this message yet - delay the enqueue + _ = Task.Delay(delay).ContinueWith(t => this.AddMessage(message)); + return; + } + + state.MessagesJson.Add(message); + + if (!state.IsLoaded) + { + // The orchestration isn't running, so schedule it to run now. + // If it is running, it will be scheduled again automatically when it's released. + this.readyToRunQueue.Schedule(state); + } + } + } + + public void AbandonInstance(IEnumerable messagesToReturn) + { + foreach (var message in messagesToReturn) + { + this.AddMessage(message); + } + } + + public void ReleaseLock(string instanceId) + { + if (!this.store.TryGetValue(instanceId, out SerializedInstanceState? state) || !state.IsLoaded) + { + throw new InvalidOperationException($"Instance {instanceId} is not in the store or is not loaded!"); + } + + lock (state) + { + state.IsLoaded = false; + if (state.MessagesJson.Count > 0) + { + // More messages came in while we were running. Or, messages were abandoned. + // Put this back into the read-to-run queue! + this.readyToRunQueue.Schedule(state); + } + } + } + + public Task WaitForInstanceAsync(string instanceId, CancellationToken cancellationToken) + { + if (this.store.TryGetValue(instanceId, out SerializedInstanceState? state)) + { + lock (state) + { + OrchestrationState? statusRecord = state.StatusRecordJson?.GetValue(); + if (statusRecord != null) + { + if (statusRecord.OrchestrationStatus == OrchestrationStatus.Completed || + statusRecord.OrchestrationStatus == OrchestrationStatus.Failed || + statusRecord.OrchestrationStatus == OrchestrationStatus.Terminated) + { + // orchestration has already completed + return Task.FromResult(statusRecord); + } + } + } + } + + // Caller will be notified when the instance completes. + // The ContinueWith is just to enable cancellation: https://stackoverflow.com/a/25652873/2069 + var tcs = this.waiters.GetOrAdd(instanceId, _ => new TaskCompletionSource()); + return tcs.Task.ContinueWith(t => t.GetAwaiter().GetResult(), cancellationToken); + } + + public OrchestrationQueryResult GetOrchestrationWithQuery(OrchestrationQuery query) + { + int startIndex = 0; + int counter = 0; + string? continuationToken = query.ContinuationToken; + if (continuationToken != null) + { + if (!Int32.TryParse(continuationToken, out startIndex)) + { + throw new InvalidOperationException($"{continuationToken} cannot be parsed to Int32"); + } + } + + counter = startIndex; + + List results = this.store + .Skip(startIndex) + .Where(item => + { + counter++; + OrchestrationState? statusRecord = item.Value.StatusRecordJson?.GetValue(); + if (statusRecord == null) + { + return false; + } + + if (query.CreatedTimeFrom != null && query.CreatedTimeFrom > statusRecord.CreatedTime) + { + return false; + } + + if (query.CreatedTimeTo != null && query.CreatedTimeTo < statusRecord.CreatedTime) + { + return false; + } + + if (query.RuntimeStatus != null && query.RuntimeStatus.Count != 0 && !query.RuntimeStatus.Contains(statusRecord.OrchestrationStatus)) + { + return false; + } + + // CA1310: Use StringComparison for culture-sensitive operations + if (query.InstanceIdPrefix != null && !statusRecord.OrchestrationInstance.InstanceId.StartsWith(query.InstanceIdPrefix, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return true; + }) + .Take(query.PageSize) + .Select(item => item.Value.StatusRecordJson!.GetValue()) + .ToList(); + + string? token = null; + if (results.Count == query.PageSize) + { + // CA1305: Use IFormatProvider + token = counter.ToString(CultureInfo.InvariantCulture); + } + + return new OrchestrationQueryResult(results, token); + } + + public PurgeResult PurgeInstanceState(string instanceId) + { + if (instanceId != null && this.store.TryGetValue(instanceId, out SerializedInstanceState? state) && state.IsCompleted) + { + this.store.TryRemove(instanceId, out SerializedInstanceState? removedState); + if (removedState != null) + { + return new PurgeResult(1); + } + } + return new PurgeResult(0); + } + + public PurgeResult PurgeInstanceState(PurgeInstanceFilter purgeInstanceFilter) + { + int counter = 0; + + List filteredInstanceIds = this.store + .Where(item => + { + OrchestrationState? statusRecord = item.Value.StatusRecordJson?.GetValue(); + if (statusRecord == null) return false; + if (purgeInstanceFilter.CreatedTimeFrom > statusRecord.CreatedTime) return false; + if (purgeInstanceFilter.CreatedTimeTo != null && purgeInstanceFilter.CreatedTimeTo < statusRecord.CreatedTime) return false; + if (purgeInstanceFilter.RuntimeStatus != null && purgeInstanceFilter.RuntimeStatus.Any() && !purgeInstanceFilter.RuntimeStatus.Contains(statusRecord.OrchestrationStatus)) return false; + return true; + }) + .Select(item => item.Key) + .ToList(); + + foreach (string instanceId in filteredInstanceIds) + { + this.store.TryRemove(instanceId, out SerializedInstanceState? removedState); + if (removedState != null) + { + counter++; + } + } + + return new PurgeResult(counter); + } + + class ReadyToRunQueue + { + readonly Channel readyToRunQueue = Channel.CreateUnbounded(); + readonly Dictionary readyInstances = new(StringComparer.OrdinalIgnoreCase); + + public void Reset() + { + this.readyInstances.Clear(); + } + + public async ValueTask TakeNextAsync(CancellationToken ct) + { + while (true) + { + SerializedInstanceState state = await this.readyToRunQueue.Reader.ReadAsync(ct); + lock (state) + { + if (this.readyInstances.Remove(state.InstanceId)) + { + if (state.IsLoaded) + { + throw new InvalidOperationException("Should never load state that is already loaded."); + } + + state.IsLoaded = true; + return state; + } + } + } + } + + public void Schedule(SerializedInstanceState state) + { + // TODO: There is a race condition here. If another thread is calling TakeNextAsync + // and removed the queue item before updating the dictionary, then we'll fail + // to update the readyToRunQueue and the orchestration will get stuck. + if (this.readyInstances.TryAdd(state.InstanceId, state)) + { + if (!this.readyToRunQueue.Writer.TryWrite(state)) + { + throw new InvalidOperationException($"unable to write to queue for {state.InstanceId}"); + } + } + } + } + + class SerializedInstanceState + { + public SerializedInstanceState(string instanceId, string? executionId) + { + this.InstanceId = instanceId; + this.ExecutionId = executionId; + } + + public string InstanceId { get; } + public string? ExecutionId { get; internal set; } + public JsonValue? StatusRecordJson { get; set; } + public JsonArray HistoryEventsJson { get; } = new JsonArray(); + public JsonArray MessagesJson { get; } = new JsonArray(); + + internal bool IsLoaded { get; set; } + internal bool IsCompleted { get; set; } + } + } +} + diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Logs.cs b/src/InProcessTestHost/Sidecar/Logs.cs similarity index 96% rename from test/Grpc.IntegrationTests/GrpcSidecar/Logs.cs rename to src/InProcessTestHost/Sidecar/Logs.cs index 3a38bc7c..c0a0eb75 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Logs.cs +++ b/src/InProcessTestHost/Sidecar/Logs.cs @@ -1,12 +1,15 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using DurableTask.Core; using DurableTask.Core.Command; using Microsoft.Extensions.Logging; -namespace Microsoft.DurableTask.Sidecar +namespace Microsoft.DurableTask.Testing.Sidecar { + /// + /// Logger message definitions for the sidecar components. + /// static partial class Logs { [LoggerMessage( @@ -118,4 +121,4 @@ public static partial void IgnoringUnknownOrchestratorAction( string instanceId, OrchestratorActionType action); } -} \ No newline at end of file +} diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Utils.cs b/src/InProcessTestHost/Sidecar/Utils.cs similarity index 76% rename from test/Grpc.IntegrationTests/GrpcSidecar/Utils.cs rename to src/InProcessTestHost/Sidecar/Utils.cs index 4be9aa6a..0c6c632f 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Utils.cs +++ b/src/InProcessTestHost/Sidecar/Utils.cs @@ -1,12 +1,21 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. using DurableTask.Core.History; -namespace Microsoft.DurableTask.Sidecar; +namespace Microsoft.DurableTask.Testing.Sidecar; -internal static class Utils +/// +/// Utility methods for the sidecar components. +/// +static class Utils { + /// + /// Tries to get the task scheduled ID from a history event. + /// + /// The history event. + /// The task scheduled ID if found. + /// True if the task scheduled ID was found, false otherwise. public static bool TryGetTaskScheduledId(HistoryEvent historyEvent, out int taskScheduledId) { switch (historyEvent.EventType) @@ -39,6 +48,7 @@ public static bool TryGetTaskScheduledId(HistoryEvent historyEvent, out int task taskScheduledId = -1; return false; } + default: taskScheduledId = -1; return false; diff --git a/test/Grpc.IntegrationTests/ClassSyntaxIntegrationTests.cs b/test/Grpc.IntegrationTests/ClassSyntaxIntegrationTests.cs new file mode 100644 index 00000000..7147fb2a --- /dev/null +++ b/test/Grpc.IntegrationTests/ClassSyntaxIntegrationTests.cs @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Worker; +using Xunit.Abstractions; +using static Microsoft.DurableTask.Grpc.Tests.ClassSyntaxTestOrchestration; + +namespace Microsoft.DurableTask.Grpc.Tests; + +/// +/// Integration tests for class-based syntax orchestrators and activities. +/// These tests demonstrate how to use TaskOrchestrator and TaskActivity base classes +/// instead of function-based syntax. +/// +public class ClassSyntaxIntegrationTests : IntegrationTestBase +{ + public ClassSyntaxIntegrationTests(ITestOutputHelper output, GrpcSidecarFixture sidecarFixture) + : base(output, sidecarFixture) + { } + + /// + /// Tests a basic orchestration that calls activities in sequence. + /// + [Fact] + public async Task HelloSequenceClassBased() + { + // Register orchestrator and activity + await using HostTestLifetime server = await this.StartWorkerAsync(b => + { + b.AddTasks(tasks => + { + tasks.AddOrchestrator(); + tasks.AddActivity(); + }); + }); + + // Schedule and wait for the orchestration to complete + string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync("HelloSequence"); + OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync( + instanceId, getInputsAndOutputs: true, this.TimeoutToken); + + // Verify the orchestration completed successfully + Assert.NotNull(metadata); + Assert.Equal(instanceId, metadata.InstanceId); + Assert.Equal(OrchestrationRuntimeStatus.Completed, metadata.RuntimeStatus); + + // Verify the output + Assert.NotNull(metadata.SerializedOutput); + List? result = metadata.ReadOutputAs>(); + Assert.NotNull(result); + Assert.Equal("Hello Tokyo!", result[0]); + Assert.Equal("Hello London!", result[1]); + Assert.Equal("Hello Seattle!", result[2]); + } + + /// + /// Tests exception handling. + /// + [Fact] + public async Task ClassBasedOrchestratorException() + { + await using HostTestLifetime server = await this.StartWorkerAsync(b => + { + b.AddTasks(tasks => + { + tasks.AddOrchestrator(); + tasks.AddActivity(); + }); + }); + + string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync("FaultyOrchestrator"); + OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync( + instanceId, getInputsAndOutputs: true, this.TimeoutToken); + + Assert.NotNull(metadata); + Assert.Equal(OrchestrationRuntimeStatus.Failed, metadata.RuntimeStatus); + Assert.NotNull(metadata.FailureDetails); + Assert.Contains("Intentional failure", metadata.FailureDetails.ErrorMessage); + } + + /// + /// Tests activity retry using RetryPolicy. + /// + [Fact] + public async Task ClassBasedActivityWithRetryPolicy() + { + await using HostTestLifetime server = await this.StartWorkerAsync(b => + { + b.AddTasks(tasks => + { + tasks.AddOrchestrator(); + tasks.AddActivity(); + }); + }); + + string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync("RetryOrchestrator"); + OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync( + instanceId, getInputsAndOutputs: true, this.TimeoutToken); + + Assert.NotNull(metadata); + Assert.Equal(OrchestrationRuntimeStatus.Completed, metadata.RuntimeStatus); + Assert.NotNull(metadata.SerializedOutput); + string? result = metadata.ReadOutputAs(); + Assert.Equal("Success after retries", result); + } + + /// + /// Tests sub-orchestration. + /// + [Fact] + public async Task ClassBasedSubOrchestration() + { + await using HostTestLifetime server = await this.StartWorkerAsync(b => + { + b.AddTasks(tasks => + { + tasks.AddOrchestrator(); + tasks.AddOrchestrator(); + tasks.AddActivity(); + }); + }); + + string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync("ParentOrchestrator", input: 5); + OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync( + instanceId, getInputsAndOutputs: true, this.TimeoutToken); + + Assert.NotNull(metadata); + Assert.Equal(OrchestrationRuntimeStatus.Completed, metadata.RuntimeStatus); + Assert.NotNull(metadata.SerializedOutput); + int result = metadata.ReadOutputAs(); + Assert.Equal(15, result); // 5 + 10 (from child) + } + + /// + /// Tests external event handling. + /// + [Fact] + public async Task ClassBasedExternalEvent() + { + await using HostTestLifetime server = await this.StartWorkerAsync(b => + { + b.AddTasks(tasks => tasks.AddOrchestrator()); + }); + + string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync("ExternalEventOrchestrator"); + + // Wait for the orchestration to start + await server.Client.WaitForInstanceStartAsync(instanceId, this.TimeoutToken); + + // Send an external event + await server.Client.RaiseEventAsync(instanceId, "ApprovalEvent", "Approved"); + + OrchestrationMetadata metadata = await server.Client.WaitForInstanceCompletionAsync( + instanceId, getInputsAndOutputs: true, this.TimeoutToken); + + Assert.NotNull(metadata); + Assert.Equal(OrchestrationRuntimeStatus.Completed, metadata.RuntimeStatus); + Assert.NotNull(metadata.SerializedOutput); + string? result = metadata.ReadOutputAs(); + Assert.Equal("Received: Approved", result); + } +} diff --git a/test/Grpc.IntegrationTests/ClassSyntaxTestOrchestration.cs b/test/Grpc.IntegrationTests/ClassSyntaxTestOrchestration.cs new file mode 100644 index 00000000..eb139c48 --- /dev/null +++ b/test/Grpc.IntegrationTests/ClassSyntaxTestOrchestration.cs @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask; + +namespace Microsoft.DurableTask.Grpc.Tests; + +/// +/// Class-based orchestrator and activity implementations for integration tests. +/// +public static class ClassSyntaxTestOrchestration +{ + /// + /// Orchestrator that calls multiple activities in sequence. + /// + [DurableTask("HelloSequence")] + public class HelloSequenceOrchestrator : TaskOrchestrator> + { + public override async Task> RunAsync(TaskOrchestrationContext context, string? input) + { + List greetings = + [ + await context.CallActivityAsync("SayHello", "Tokyo"), + await context.CallActivityAsync("SayHello", "London"), + await context.CallActivityAsync("SayHello", "Seattle"), + ]; + + return greetings; + } + } + + /// + /// Orchestrator that calls a activity which will throw exception. + /// + [DurableTask("FaultyOrchestrator")] + public class FaultyOrchestrator : TaskOrchestrator + { + public override async Task RunAsync(TaskOrchestrationContext context, string? input) + { + await context.CallActivityAsync("ThrowingActivity"); + return null; + } + } + + /// + /// Orchestrator that calls activities with retry option. + [DurableTask("RetryOrchestrator")] + public class RetryOrchestrator : TaskOrchestrator + { + public override async Task RunAsync(TaskOrchestrationContext context, string? input) + { + // Create retry policy: 3 attempts with 1ms interval + var retryPolicy = new RetryPolicy( + maxNumberOfAttempts: 3, + firstRetryInterval: TimeSpan.FromMilliseconds(1)); + + var options = TaskOptions.FromRetryPolicy(retryPolicy); + + string result = await context.CallActivityAsync("RetryableActivity", options: options); + return result; + } + } + + /// + /// Orchestrator that calls a sub-orchestrator. + /// + [DurableTask("ParentOrchestrator")] + public class ParentOrchestrator : TaskOrchestrator + { + public override async Task RunAsync(TaskOrchestrationContext context, int input) + { + int childResult = await context.CallSubOrchestratorAsync("ChildOrchestrator", input: input); + return input + childResult; + } + } + + /// + /// Test sub-orchestrations. + /// + [DurableTask("ChildOrchestrator")] + public class ChildOrchestrator : TaskOrchestrator + { + public override async Task RunAsync(TaskOrchestrationContext context, int input) + { + int result = await context.CallActivityAsync("ChildActivity", input); + return result; + } + } + + /// + /// Orchestrator that waits for external event. + /// + [DurableTask("ExternalEventOrchestrator")] + public class ExternalEventOrchestrator : TaskOrchestrator + { + public override async Task RunAsync(TaskOrchestrationContext context, string? input) + { + string eventData = await context.WaitForExternalEvent("ApprovalEvent"); + return $"Received: {eventData}"; + } + } + + [DurableTask("SayHello")] + public class SayHelloActivity : TaskActivity + { + public override Task RunAsync(TaskActivityContext context, string city) + { + return Task.FromResult($"Hello {city}!"); + } + } + + /// + /// Activity that throws an exception. + /// + [DurableTask("ThrowingActivity")] + public class ThrowingActivity : TaskActivity + { + public override Task RunAsync(TaskActivityContext context, string? input) + { + throw new InvalidOperationException("Intentional failure."); + } + } + + /// + /// Activity that throws exception on first attempt but succeed with third attemp. + /// + [DurableTask("RetryableActivity")] + public class RetryableActivity : TaskActivity + { + static int attemptCount = 0; + + public override Task RunAsync(TaskActivityContext context, string? input) + { + int currentAttempt = Interlocked.Increment(ref attemptCount); + + // Fail on first two attempts, succeed on third + if (currentAttempt < 3) + { + throw new InvalidOperationException($"Attempt {currentAttempt} failed"); + } + + return Task.FromResult("Success after retries"); + } + } + + [DurableTask("ChildActivity")] + public class ChildActivity : TaskActivity + { + public override Task RunAsync(TaskActivityContext context, int input) + { + return Task.FromResult(input * 2); + } + } +} + diff --git a/test/Grpc.IntegrationTests/Grpc.IntegrationTests.csproj b/test/Grpc.IntegrationTests/Grpc.IntegrationTests.csproj index ba2c8b68..88ab0633 100644 --- a/test/Grpc.IntegrationTests/Grpc.IntegrationTests.csproj +++ b/test/Grpc.IntegrationTests/Grpc.IntegrationTests.csproj @@ -5,13 +5,8 @@ - - - - - - + diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs deleted file mode 100644 index 302e13d4..00000000 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcOrchestratorExecutionResult.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using DurableTask.Core; - -namespace Microsoft.DurableTask.Sidecar.Dispatcher; - -public class GrpcOrchestratorExecutionResult : OrchestratorExecutionResult -{ - public string? OrchestrationActivitySpanId { get; set; } - public DateTimeOffset? OrchestrationActivityStartTime { get; set; } -} diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcScheduleTaskOrchestratorAction.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcScheduleTaskOrchestratorAction.cs deleted file mode 100644 index bd8a5af4..00000000 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcScheduleTaskOrchestratorAction.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using DurableTask.Core.Command; -using DurableTask.Core.Tracing; - -namespace Microsoft.DurableTask.Sidecar.Dispatcher; - -public class GrpcScheduleTaskOrchestratorAction : ScheduleTaskOrchestratorAction -{ - public DistributedTraceContext? ParentTraceContext { get; set; } -} diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs deleted file mode 100644 index 466f891c..00000000 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/GrpcSubOrchestrationInstanceCreatedEvent.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Runtime.Serialization; -using DurableTask.Core.History; -using DurableTask.Core.Tracing; - -namespace Microsoft.DurableTask.Sidecar.Dispatcher; - -public class GrpcSubOrchestrationInstanceCreatedEvent : SubOrchestrationInstanceCreatedEvent -{ - public GrpcSubOrchestrationInstanceCreatedEvent(int eventId) - : base(eventId) - { - } - - [DataMember] - public DistributedTraceContext? ParentTraceContext { get; set; } -} diff --git a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskActivityDispatcher.cs b/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskActivityDispatcher.cs deleted file mode 100644 index 282d6a9c..00000000 --- a/test/Grpc.IntegrationTests/GrpcSidecar/Dispatcher/TaskActivityDispatcher.cs +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using DurableTask.Core; -using DurableTask.Core.History; -using Microsoft.Extensions.Logging; - -namespace Microsoft.DurableTask.Sidecar.Dispatcher; - -class TaskActivityDispatcher : WorkItemDispatcher -{ - readonly IOrchestrationService service; - readonly ITaskExecutor taskExecutor; - - public TaskActivityDispatcher(ILogger log, ITrafficSignal trafficSignal, IOrchestrationService service, ITaskExecutor taskExecutor) - : base(log, trafficSignal) - { - this.service = service; - this.taskExecutor = taskExecutor; - } - - public override int MaxWorkItems => this.service.MaxConcurrentTaskActivityWorkItems; - - public override Task AbandonWorkItemAsync(TaskActivityWorkItem workItem) => - this.service.AbandonTaskActivityWorkItemAsync(workItem); - - public override Task FetchWorkItemAsync(TimeSpan timeout, CancellationToken cancellationToken) => - this.service.LockNextTaskActivityWorkItem(timeout, cancellationToken); - - protected override async Task ExecuteWorkItemAsync(TaskActivityWorkItem workItem) - { - TaskScheduledEvent scheduledEvent = (TaskScheduledEvent)workItem.TaskMessage.Event; - - // TODO: Error handling for internal errors (user code exceptions are handled by the executor). - ActivityExecutionResult result = await this.taskExecutor.ExecuteActivity( - instance: workItem.TaskMessage.OrchestrationInstance, - activityEvent: scheduledEvent); - - TaskMessage responseMessage = new() - { - Event = result.ResponseEvent, - OrchestrationInstance = workItem.TaskMessage.OrchestrationInstance, - }; - - await this.service.CompleteTaskActivityWorkItemAsync(workItem, responseMessage); - } - - public override int GetDelayInSecondsOnFetchException(Exception ex) => - this.service.GetDelayInSecondsAfterOnFetchException(ex); - - public override string GetWorkItemId(TaskActivityWorkItem workItem) => workItem.Id; - - // No-op - public override Task ReleaseWorkItemAsync(TaskActivityWorkItem workItem) => Task.CompletedTask; - - public override Task RenewWorkItemAsync(TaskActivityWorkItem workItem) => - this.service.RenewTaskActivityWorkItemLockAsync(workItem); -} \ No newline at end of file diff --git a/test/Grpc.IntegrationTests/GrpcSidecarFixture.cs b/test/Grpc.IntegrationTests/GrpcSidecarFixture.cs index 9a6d6713..2408d3d2 100644 --- a/test/Grpc.IntegrationTests/GrpcSidecarFixture.cs +++ b/test/Grpc.IntegrationTests/GrpcSidecarFixture.cs @@ -6,8 +6,8 @@ using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Server.Kestrel.Core; -using Microsoft.DurableTask.Sidecar; -using Microsoft.DurableTask.Sidecar.Grpc; +using Microsoft.DurableTask.Testing.Sidecar; +using Microsoft.DurableTask.Testing.Sidecar.Grpc; using Microsoft.DurableTask.Worker; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; diff --git a/test/Grpc.IntegrationTests/OrchestrationErrorHandling.cs b/test/Grpc.IntegrationTests/OrchestrationErrorHandling.cs index b4a687a6..84c2f1e4 100644 --- a/test/Grpc.IntegrationTests/OrchestrationErrorHandling.cs +++ b/test/Grpc.IntegrationTests/OrchestrationErrorHandling.cs @@ -735,7 +735,7 @@ void MyActivityImpl(TaskActivityContext ctx) => /// /// Tests that exception properties are included in FailureDetails when an orchestration - /// throws exception directly without calling any other functions and a custom provider is set. + /// throws ArgumentOutOfRangeException directly without calling any other functions. /// [Fact] public async Task OrchestrationDirectArgumentOutOfRangeExceptionProperties() @@ -745,6 +745,7 @@ public async Task OrchestrationDirectArgumentOutOfRangeExceptionProperties() string actualValue = "invalidValue"; string errorMessage = $"Parameter '{paramName}' is out of range."; + // Register orchestration that throws ArgumentOutOfRangeException directly void MyOrchestrationImpl(TaskOrchestrationContext ctx) => throw new ArgumentOutOfRangeException(paramName, actualValue, errorMessage); @@ -787,8 +788,8 @@ void MyOrchestrationImpl(TaskOrchestrationContext ctx) => } /// - /// Tests that exception properties are included through nested orchestration calls when - /// a provider is set. + /// Tests that exception properties are preserved through nested orchestration calls when + /// a parent orchestration calls a sub-orchestration, which then calls an activity that throws ArgumentOutOfRangeException. /// [Fact] public async Task NestedOrchestrationArgumentOutOfRangeExceptionProperties() diff --git a/test/InProcessTestHost.Tests/BasicOrchestrationTests.cs b/test/InProcessTestHost.Tests/BasicOrchestrationTests.cs new file mode 100644 index 00000000..3e14077f --- /dev/null +++ b/test/InProcessTestHost.Tests/BasicOrchestrationTests.cs @@ -0,0 +1,85 @@ +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Testing; +using Microsoft.DurableTask.Worker; +using Xunit; + +namespace InProcessTestHost.Tests; + +/// +/// Tests to verify InProcessTestHost works with both class-syntax and function-syntax orchestrations. +/// +public class BasicOrchestrationTests +{ + [Fact] + // Test basic class-syntax orchestration with DurableTaskTestHost. + public async Task TestClassSyntaxOrchestration() + { + await using DurableTaskTestHost host = await DurableTaskTestHost.StartAsync(tasks => + { + tasks.AddOrchestrator(); + tasks.AddActivity(); + }); + + var instanceId = await host.Client.ScheduleNewOrchestrationInstanceAsync(nameof(MyClassOrchestrator), "Alice"); + var metadata = await host.Client.WaitForInstanceCompletionAsync(instanceId, getInputsAndOutputs: true); + + + Assert.NotNull(metadata); + Assert.Equal(OrchestrationRuntimeStatus.Completed, metadata.RuntimeStatus); + var result = metadata.ReadOutputAs(); + Assert.Equal("Hello, Alice!", result); + } + + [Fact] + // Test basic class-syntax orchestration with DurableTaskTestHost. + public async Task TestFunctionSyntaxOrchestration() + { + + await using DurableTaskTestHost host = await DurableTaskTestHost.StartAsync(tasks => + { + tasks.AddOrchestratorFunc("MyFuncOrchestrator", async (TaskOrchestrationContext context, string name) => + { + var result = await context.CallActivityAsync("MyFuncActivity", name); + return result; + }); + + tasks.AddActivityFunc("MyFuncActivity", (TaskActivityContext context, string name) => + { + return $"Greetings, {name}!"; + }); + }); + + var instanceId = await host.Client.ScheduleNewOrchestrationInstanceAsync("MyFuncOrchestrator", "Bob"); + var metadata = await host.Client.WaitForInstanceCompletionAsync(instanceId, getInputsAndOutputs: true); + + Assert.NotNull(metadata); + Assert.Equal(OrchestrationRuntimeStatus.Completed, metadata.RuntimeStatus); + var result = metadata.ReadOutputAs(); + Assert.Equal("Greetings, Bob!", result); + } + + /// + /// Class-based orchestrator that calls an activity. + /// + class MyClassOrchestrator : TaskOrchestrator + { + public override async Task RunAsync(TaskOrchestrationContext context, string name) + { + var result = await context.CallActivityAsync(nameof(MyClassActivity), name); + return result; + } + } + + /// + /// Class-based activity that returns a greeting. + /// + class MyClassActivity : TaskActivity + { + public override Task RunAsync(TaskActivityContext context, string name) + { + return Task.FromResult($"Hello, {name}!"); + } + } +} + diff --git a/test/InProcessTestHost.Tests/InProcessTestHost.Tests.csproj b/test/InProcessTestHost.Tests/InProcessTestHost.Tests.csproj new file mode 100644 index 00000000..479d88a1 --- /dev/null +++ b/test/InProcessTestHost.Tests/InProcessTestHost.Tests.csproj @@ -0,0 +1,16 @@ + + + + net8.0 + enable + enable + + false + true + + + + + + +