diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index c37c1b5..9045ea0 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,13 +3,13 @@ "isRoot": true, "tools": { "dotnet-sonarscanner": { - "version": "5.9.2", + "version": "6.2.0", "commands": [ "dotnet-sonarscanner" ] }, "gitversion.tool": { - "version": "5.11.1", + "version": "5.12.0", "commands": [ "dotnet-gitversion" ] diff --git a/.editorconfig b/.editorconfig index b10163a..fb75f21 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,8 +8,11 @@ root = true [*] indent_style = space +[*.sln] +indent_style = tab + # Code files -[*.{cs,csx,vb,vbx}] +[*.{cs,csx,vb,vbx,fs,fsx,fsi}] indent_size = 4 max_line_length = 120 insert_final_newline = true @@ -33,3 +36,17 @@ indent_size = 4 [*.md] trim_trailing_whitespace = true insert_final_newline = true + +# See https://github.com/dotnet/aspnetcore/blob/main/.editorconfig +[src/**/*.{cs,csx,vb,vbx,fs,fsx,fsi}] + +# See https://www.jetbrains.com/help/resharper/ConfigureAwait_Analysis.html +configure_await_analysis_mode = library +# CA2007: Consider calling ConfigureAwait on the awaited task +#dotnet_diagnostic.CA2007.severity = warning + +# CA2012: Use ValueTask correctly +dotnet_diagnostic.CA2012.severity = warning + +# CA2013: Do not use ReferenceEquals with value types +dotnet_diagnostic.CA2013.severity = warning diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4c5c935..6fff16c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,11 +1,6 @@ ---- version: 2 updates: - package-ecosystem: github-actions directory: "/" schedule: - interval: weekly - - package-ecosystem: nuget - directory: "/" - schedule: - interval: weekly + interval: monthly diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index bfce883..d646d95 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -1,25 +1,24 @@ ---- name: Publish on: release: types: [ published ] + jobs: test: runs-on: ubuntu-latest env: DOTNET_NOLOGO: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Required for GitVersion fetch-depth: 0 - - uses: actions/setup-dotnet@v3 + - uses: actions/setup-dotnet@v4 with: dotnet-version: | - 3.1.x - 6.0.x - 7.0.x + 8.0.x + 9.0.x - run: dotnet restore - run: dotnet build -c Release --no-restore - run: dotnet test -c Release --no-build --verbosity=minimal @@ -29,14 +28,14 @@ jobs: env: DOTNET_NOLOGO: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Required for GitVersion fetch-depth: 0 - - uses: actions/setup-dotnet@v3 + - uses: actions/setup-dotnet@v4 with: dotnet-version: | - 7.0.x + 9.0.x - run: dotnet pack -c Release - name: Publish run: | diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml index 1f8e284..37815a1 100644 --- a/.github/workflows/qa.yml +++ b/.github/workflows/qa.yml @@ -1,4 +1,3 @@ ---- name: QA on: @@ -6,21 +5,22 @@ on: branches: [ master, main ] pull_request: branches: [ master, main ] + jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Full history is needed to get a proper list of changed files fetch-depth: 0 - - uses: github/super-linter@v4 + - uses: github/super-linter@v7 env: DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VALIDATE_ALL_CODEBASE: false # Only changed files + VALIDATE_ALL_CODEBASE: false # Only changed files VALIDATE_EDITORCONFIG: true - VALIDATE_CSHARP: true + VALIDATE_CSHARP: false # Checked by SonarQube VALIDATE_JSON: true VALIDATE_MARKDOWN: true VALIDATE_YAML: true @@ -30,20 +30,19 @@ jobs: env: DOTNET_NOLOGO: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: - # Disabling shallow clone is recommended by SonarCloud for improving relevancy of reporting + # Disabling shallow clone is recommended by SonarQube for improving relevancy of reporting fetch-depth: 0 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: distribution: temurin - java-version: 17 - - uses: actions/setup-dotnet@v3 + java-version: 21 + - uses: actions/setup-dotnet@v4 with: dotnet-version: | - 3.1.x - 6.0.x - 7.0.x + 8.0.x + 9.0.x - run: dotnet tool restore - run: dotnet gitversion /output buildserver - run: ./sonar-scan.sh diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..dffd79e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,32 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Fixed + +### Added + +### Changed + +### Removed + +## [0.2.0] - 2025-03-12 + +### Changed + +- Whole new design + +### Removed + +- LocalPost.SnsPublisher package + +## [0.1.0] - 2023-01-01 + +### Added + +- Initial release diff --git a/Directory.Build.props b/Directory.Build.props deleted file mode 100644 index 8bb8702..0000000 --- a/Directory.Build.props +++ /dev/null @@ -1,18 +0,0 @@ - - - - 11 - enable - enable - true - - - - - <_Parameter1>$(MSBuildProjectName).Tests - - - <_Parameter1>DynamicProxyGenAssembly2 - - - diff --git a/LocalPost.sln b/LocalPost.sln index 2c7c844..76f0263 100644 --- a/LocalPost.sln +++ b/LocalPost.sln @@ -2,18 +2,24 @@ Microsoft Visual Studio Solution File, Format Version 12.00 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalPost", "src\LocalPost\LocalPost.csproj", "{474D2C1A-5557-4ED9-AF20-FE195D4C1AF7}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SampleWebApp", "samples\SampleWebApp\SampleWebApp.csproj", "{46FC61E6-D0FB-4D7D-A81B-20EF8D8D1F4E}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalPost.SnsPublisher", "src\LocalPost.SnsPublisher\LocalPost.SnsPublisher.csproj", "{D256C568-2B42-4DCC-AB54-15B512A99C44}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BackgroundQueueApp", "examples\BackgroundQueueApp\BackgroundQueueApp.csproj", "{46FC61E6-D0FB-4D7D-A81B-20EF8D8D1F4E}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalPost.Tests", "tests\LocalPost.Tests\LocalPost.Tests.csproj", "{0E69A423-5F70-4BA7-8015-0AB0BC4B6FD2}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalPost.SnsPublisher.Tests", "tests\LocalPost.SnsPublisher.Tests\LocalPost.SnsPublisher.Tests.csproj", "{0B8929F4-E220-45A9-A279-41F5D94A8C1B}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalPost.SqsConsumer", "src\LocalPost.SqsConsumer\LocalPost.SqsConsumer.csproj", "{30232703-C103-4F7A-9822-80F2F680A88D}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalPost.SqsConsumer.Tests", "tests\LocalPost.SqsConsumer.Tests\LocalPost.SqsConsumer.Tests.csproj", "{2F61DCD7-E4CB-4ECC-B24E-A663D12D9C03}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalPost.KafkaConsumer", "src\LocalPost.KafkaConsumer\LocalPost.KafkaConsumer.csproj", "{D9139C53-5B9F-49E7-80DF-41C995C37E2F}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Examples", "Examples", "{405721DC-F290-4191-B638-9907D5EB042B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KafkaConsumerApp", "examples\KafkaConsumerApp\KafkaConsumerApp.csproj", "{C310487A-B976-4D3E-80AF-4ADBE1C63139}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SqsConsumerApp", "examples\SqsConsumerApp\SqsConsumerApp.csproj", "{2778AEBD-0345-4F79-9E93-73AFAB6C7BCD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LocalPost.KafkaConsumer.Tests", "tests\LocalPost.KafkaConsumer.Tests\LocalPost.KafkaConsumer.Tests.csproj", "{734C9C76-B3D8-4AD7-8E76-B14539C3CB4D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -28,18 +34,10 @@ Global {46FC61E6-D0FB-4D7D-A81B-20EF8D8D1F4E}.Debug|Any CPU.Build.0 = Debug|Any CPU {46FC61E6-D0FB-4D7D-A81B-20EF8D8D1F4E}.Release|Any CPU.ActiveCfg = Release|Any CPU {46FC61E6-D0FB-4D7D-A81B-20EF8D8D1F4E}.Release|Any CPU.Build.0 = Release|Any CPU - {D256C568-2B42-4DCC-AB54-15B512A99C44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D256C568-2B42-4DCC-AB54-15B512A99C44}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D256C568-2B42-4DCC-AB54-15B512A99C44}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D256C568-2B42-4DCC-AB54-15B512A99C44}.Release|Any CPU.Build.0 = Release|Any CPU {0E69A423-5F70-4BA7-8015-0AB0BC4B6FD2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {0E69A423-5F70-4BA7-8015-0AB0BC4B6FD2}.Debug|Any CPU.Build.0 = Debug|Any CPU {0E69A423-5F70-4BA7-8015-0AB0BC4B6FD2}.Release|Any CPU.ActiveCfg = Release|Any CPU {0E69A423-5F70-4BA7-8015-0AB0BC4B6FD2}.Release|Any CPU.Build.0 = Release|Any CPU - {0B8929F4-E220-45A9-A279-41F5D94A8C1B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {0B8929F4-E220-45A9-A279-41F5D94A8C1B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0B8929F4-E220-45A9-A279-41F5D94A8C1B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {0B8929F4-E220-45A9-A279-41F5D94A8C1B}.Release|Any CPU.Build.0 = Release|Any CPU {30232703-C103-4F7A-9822-80F2F680A88D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {30232703-C103-4F7A-9822-80F2F680A88D}.Debug|Any CPU.Build.0 = Debug|Any CPU {30232703-C103-4F7A-9822-80F2F680A88D}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -48,5 +46,26 @@ Global {2F61DCD7-E4CB-4ECC-B24E-A663D12D9C03}.Debug|Any CPU.Build.0 = Debug|Any CPU {2F61DCD7-E4CB-4ECC-B24E-A663D12D9C03}.Release|Any CPU.ActiveCfg = Release|Any CPU {2F61DCD7-E4CB-4ECC-B24E-A663D12D9C03}.Release|Any CPU.Build.0 = Release|Any CPU + {D9139C53-5B9F-49E7-80DF-41C995C37E2F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D9139C53-5B9F-49E7-80DF-41C995C37E2F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D9139C53-5B9F-49E7-80DF-41C995C37E2F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D9139C53-5B9F-49E7-80DF-41C995C37E2F}.Release|Any CPU.Build.0 = Release|Any CPU + {C310487A-B976-4D3E-80AF-4ADBE1C63139}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C310487A-B976-4D3E-80AF-4ADBE1C63139}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C310487A-B976-4D3E-80AF-4ADBE1C63139}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C310487A-B976-4D3E-80AF-4ADBE1C63139}.Release|Any CPU.Build.0 = Release|Any CPU + {2778AEBD-0345-4F79-9E93-73AFAB6C7BCD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2778AEBD-0345-4F79-9E93-73AFAB6C7BCD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2778AEBD-0345-4F79-9E93-73AFAB6C7BCD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2778AEBD-0345-4F79-9E93-73AFAB6C7BCD}.Release|Any CPU.Build.0 = Release|Any CPU + {734C9C76-B3D8-4AD7-8E76-B14539C3CB4D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {734C9C76-B3D8-4AD7-8E76-B14539C3CB4D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {734C9C76-B3D8-4AD7-8E76-B14539C3CB4D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {734C9C76-B3D8-4AD7-8E76-B14539C3CB4D}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {46FC61E6-D0FB-4D7D-A81B-20EF8D8D1F4E} = {405721DC-F290-4191-B638-9907D5EB042B} + {C310487A-B976-4D3E-80AF-4ADBE1C63139} = {405721DC-F290-4191-B638-9907D5EB042B} + {2778AEBD-0345-4F79-9E93-73AFAB6C7BCD} = {405721DC-F290-4191-B638-9907D5EB042B} EndGlobalSection EndGlobal diff --git a/README.md b/README.md index 7ecd402..b7c09c9 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,53 @@ # LocalPost +[![NuGet package](https://img.shields.io/nuget/dt/LocalPost)](https://www.nuget.org/packages/LocalPost/) +[![Code coverage](https://img.shields.io/sonar/coverage/alexeyshockov_LocalPost.NET?server=https%3A%2F%2Fsonarcloud.io)](https://sonarcloud.io/project/overview?id=alexeyshockov_LocalPost.NET) + Simple .NET in-memory background queue ([System.Threading.Channels](https://learn.microsoft.com/de-de/dotnet/api/system.threading.channels?view=net-6.0) based). -## Alternatives +## Background tasks + +There are multiple ways to run background tasks in .NET. The most common are: + +## Usage + +### Installation + +For the core library: + +```shell +dotnet add package LocalPost +``` + +AWS SQS, Kafka and other integrations are provided as separate packages, like: + +```shell +dotnet add package LocalPost.SqsConsumer +dotnet add package LocalPost.KafkaConsumer +``` + +### .NET 8 asynchronous background services handling + +Before version 8 .NET runtime handled start/stop of the services only synchronously, but now it is possible to enable +concurrent handling of the services. This is done by setting `HostOptions` property `ConcurrentServiceExecution` +to `true`: + +See for details: +- https://github.com/dotnet/runtime/blob/v8.0.0/src/libraries/Microsoft.Extensions.Hosting/src/Internal/Host.cs +- https://github.com/dotnet/runtime/blob/main/src/libraries/Microsoft.Extensions.Hosting/src/HostOptions.cs + +## Similar projects + +- [Coravel queue](https://docs.coravel.net/Queuing/) — a simple job queue + +More complex jobs management / scheduling: +- [Hangfire](https://www.hangfire.io/) — background job scheduler. Supports advanced scheduling, persistence and jobs distribution across multiple workers. + +Service bus (for bigger solutions): +- [JustSaying](https://github.com/justeattakeaway/JustSaying) +- [NServiceBus](https://docs.particular.net/nservicebus/) +- [MassTransit](https://masstransit.io/) + +## Inspiration -- [Coravel queue](https://docs.coravel.net/Queuing/)/event broadcasting — only invocable queueing, event broadcasting is different from consuming a queue -- [Hangfire](https://www.hangfire.io/) — for persistent queues (means payload serialisation), LocalPost is completely about in-memory ones +- [FastStream](https://github.com/airtai/faststream) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..2d3f7a3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,105 @@ +name: localpost +networks: + redpanda_network: + driver: bridge +volumes: + redpanda: + driver: local + localstack: + driver: local +services: + # https://learn.microsoft.com/en-us/dotnet/aspire/fundamentals/dashboard/standalone + # https://hub.docker.com/r/microsoft/dotnet-aspire-dashboard + aspire: + image: mcr.microsoft.com/dotnet/aspire-dashboard:9.0 + ports: + - "18888:18888" # HTTP + - "18889:18889" # OTEL collector GRPC + - "18890:18890" # OTEL collector HTTP + environment: + - DOTNET_DASHBOARD_UNSECURED_ALLOW_ANONYMOUS=true +# - Dashboard__Otlp__AuthMode=Unsecured + # This setting is a shortcut to configuring Dashboard:Frontend:AuthMode and Dashboard:Otlp:AuthMode to Unsecured + - ASPIRE_ALLOW_UNSECURED_TRANSPORT=true + localstack: + # https://docs.localstack.cloud/getting-started/installation/#docker-compose + image: localstack/localstack:4 + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4510-4559:4510-4559" # External services port range + environment: + # LocalStack configuration: https://docs.localstack.cloud/references/configuration/ + - DEBUG=${DEBUG:-0} + - SERVICES=sqs + volumes: + # Local volume + - localstack:/var/lib/localstack + # Fixtures, see https://docs.localstack.cloud/references/init-hooks/ + - ./localstack/init/ready.d:/etc/localstack/init/ready.d" # SQS hooks + # Only needed for Lambdas +# - /var/run/docker.sock:/var/run/docker.sock + redpanda: + # Mainly from: https://docs.redpanda.com/redpanda-labs/docker-compose/single-broker/ + # See also: https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/docker-image/ + image: docker.redpanda.com/redpandadata/redpanda:v24.3.3 + container_name: redpanda + command: + - redpanda start + - --mode dev-container + - --smp 1 + - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092 + # Address the broker advertises to clients that connect to the Kafka API. + # Use the internal addresses to connect to the Redpanda brokers + # from inside the same Docker network. + # Use the external addresses to connect to the Redpanda brokers + # from outside the Docker network. + - --advertise-kafka-addr internal://redpanda:9092,external://127.0.0.1:19092 + - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082 + # Address the broker advertises to clients that connect to the HTTP Proxy. + - --advertise-pandaproxy-addr internal://redpanda:8082,external://127.0.0.1:18082 + - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081 + # Redpanda brokers use the RPC API to communicate with each other internally. + - --rpc-addr redpanda:33145 + - --advertise-rpc-addr redpanda:33145 + ports: + - "18081:18081" + - "18082:18082" + - "19092:19092" + - "19644:9644" + volumes: + - redpanda:/var/lib/redpanda/data + networks: + - redpanda_network +# healthcheck: +# test: [ "CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1" ] +# interval: 15s +# timeout: 3s +# retries: 5 +# start_period: 5s + redpanda-console: + image: docker.redpanda.com/redpandadata/console:v2.8.2 + entrypoint: /bin/sh + command: -c "echo \"$$CONSOLE_CONFIG_FILE\" > /tmp/config.yml; /app/console" + environment: + CONFIG_FILEPATH: /tmp/config.yml + CONSOLE_CONFIG_FILE: | + kafka: + brokers: ["redpanda:9092"] + schemaRegistry: + enabled: true + urls: ["http://redpanda:8081"] + redpanda: + adminApi: + enabled: true + urls: ["http://redpanda:9644"] + connect: + enabled: true + clusters: + - name: local-connect-cluster + url: http://connect:8083 + ports: + - "8080:8080" + networks: + - redpanda_network + depends_on: + - redpanda diff --git a/examples/BackgroundQueueApp/BackgroundQueueApp.csproj b/examples/BackgroundQueueApp/BackgroundQueueApp.csproj new file mode 100644 index 0000000..b0638c4 --- /dev/null +++ b/examples/BackgroundQueueApp/BackgroundQueueApp.csproj @@ -0,0 +1,26 @@ + + + + net8 + + + + + + + + + + + + + + + + + + appsettings.json + + + + diff --git a/samples/SampleWebApp/Controllers/WeatherForecastController.cs b/examples/BackgroundQueueApp/Controllers/WeatherForecastController.cs similarity index 51% rename from samples/SampleWebApp/Controllers/WeatherForecastController.cs rename to examples/BackgroundQueueApp/Controllers/WeatherForecastController.cs index 05c8458..c3e88e1 100644 --- a/samples/SampleWebApp/Controllers/WeatherForecastController.cs +++ b/examples/BackgroundQueueApp/Controllers/WeatherForecastController.cs @@ -1,27 +1,16 @@ -using Amazon.SimpleNotificationService.Model; using LocalPost; -using LocalPost.SnsPublisher; using Microsoft.AspNetCore.Mvc; -namespace SampleWebApp.Controllers; +namespace BackgroundQueueApp.Controllers; [ApiController] [Route("[controller]")] -public class WeatherForecastController : ControllerBase +public class WeatherForecastController(IBackgroundQueue queue) : ControllerBase { private static readonly string[] Summaries = - { + [ "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching" - }; - - private readonly IBackgroundQueue _queue; - private readonly ISnsPublisher _sns; - - public WeatherForecastController(IBackgroundQueue queue, ISnsPublisher sns) - { - _queue = queue; - _sns = sns; - } + ]; [HttpGet(Name = "GetWeatherForecast")] public async ValueTask> Get() @@ -33,12 +22,7 @@ public async ValueTask> Get() Summary = Summaries[Random.Shared.Next(Summaries.Length)] }).ToArray(); - await _queue.Enqueue(forecasts[0]); - - await _sns.ForTopic("arn:aws:sns:eu-central-1:703886664977:test").Enqueue(new PublishBatchRequestEntry - { - Message = forecasts[0].Summary - }); + await queue.Enqueue(forecasts[0]); return forecasts; } diff --git a/examples/BackgroundQueueApp/Program.cs b/examples/BackgroundQueueApp/Program.cs new file mode 100644 index 0000000..b3247e9 --- /dev/null +++ b/examples/BackgroundQueueApp/Program.cs @@ -0,0 +1,53 @@ +using BackgroundQueueApp; +using LocalPost; +using LocalPost.BackgroundQueue; +using LocalPost.BackgroundQueue.DependencyInjection; +using LocalPost.Resilience; +using Polly; +using Polly.Retry; + +var builder = WebApplication.CreateBuilder(args); + +// See https://github.com/App-vNext/Polly/blob/main/docs/migration-v8.md +var resiliencePipeline = new ResiliencePipelineBuilder() + .AddRetry(new RetryStrategyOptions + { + MaxRetryAttempts = 3, + Delay = TimeSpan.FromSeconds(1), + BackoffType = DelayBackoffType.Constant, + ShouldHandle = new PredicateBuilder().Handle() + }) + .AddTimeout(TimeSpan.FromSeconds(3)) + .Build(); + +// A background queue with an inline handler +builder.Services.AddBackgroundQueues(bq => + bq.AddQueue(HandlerStack.For(async (weather, ct) => + { + await Task.Delay(TimeSpan.FromSeconds(2), ct); + Console.WriteLine(weather.Summary); + }) + .UseMessagePayload() + .Scoped() + .Trace() + .UsePollyPipeline(resiliencePipeline) + .LogExceptions() + ) +); + +builder.Services.AddControllers(); +// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle +builder.Services.AddEndpointsApiExplorer(); +builder.Services.AddSwaggerGen(); + +var app = builder.Build(); +if (app.Environment.IsDevelopment()) +{ + app.UseSwagger(); + app.UseSwaggerUI(); +} + +app.UseHttpsRedirection(); +app.UseAuthorization(); +app.MapControllers(); +app.Run(); diff --git a/samples/SampleWebApp/Properties/launchSettings.json b/examples/BackgroundQueueApp/Properties/launchSettings.json similarity index 81% rename from samples/SampleWebApp/Properties/launchSettings.json rename to examples/BackgroundQueueApp/Properties/launchSettings.json index af9bed4..70edaa4 100644 --- a/samples/SampleWebApp/Properties/launchSettings.json +++ b/examples/BackgroundQueueApp/Properties/launchSettings.json @@ -8,8 +8,7 @@ "launchUrl": "swagger", "applicationUrl": "https://localhost:7003;http://localhost:5103", "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development", - "AWS_PROFILE": "kw-test" + "ASPNETCORE_ENVIRONMENT": "Development" } } } diff --git a/samples/SampleWebApp/WeatherForecast.cs b/examples/BackgroundQueueApp/WeatherForecast.cs similarity index 88% rename from samples/SampleWebApp/WeatherForecast.cs rename to examples/BackgroundQueueApp/WeatherForecast.cs index 72eee54..a7947f4 100644 --- a/samples/SampleWebApp/WeatherForecast.cs +++ b/examples/BackgroundQueueApp/WeatherForecast.cs @@ -1,4 +1,4 @@ -namespace SampleWebApp; +namespace BackgroundQueueApp; public class WeatherForecast { diff --git a/samples/SampleWebApp/appsettings.Development.json b/examples/BackgroundQueueApp/appsettings.Development.json similarity index 100% rename from samples/SampleWebApp/appsettings.Development.json rename to examples/BackgroundQueueApp/appsettings.Development.json diff --git a/samples/SampleWebApp/appsettings.json b/examples/BackgroundQueueApp/appsettings.json similarity index 100% rename from samples/SampleWebApp/appsettings.json rename to examples/BackgroundQueueApp/appsettings.json diff --git a/examples/Directory.Build.props b/examples/Directory.Build.props new file mode 100644 index 0000000..aad1d7c --- /dev/null +++ b/examples/Directory.Build.props @@ -0,0 +1,16 @@ + + + + 13 + enable + enable + true + + false + + + + + + + diff --git a/examples/KafkaConsumerApp/KafkaConsumerApp.csproj b/examples/KafkaConsumerApp/KafkaConsumerApp.csproj new file mode 100644 index 0000000..d25a35e --- /dev/null +++ b/examples/KafkaConsumerApp/KafkaConsumerApp.csproj @@ -0,0 +1,32 @@ + + + + net8 + + + + + + + + + + + + + + + + + + + + + + + + appsettings.json + + + + diff --git a/examples/KafkaConsumerApp/Program.cs b/examples/KafkaConsumerApp/Program.cs new file mode 100644 index 0000000..2cd5b02 --- /dev/null +++ b/examples/KafkaConsumerApp/Program.cs @@ -0,0 +1,55 @@ +using System.Text.Json; +using Confluent.Kafka; +using JetBrains.Annotations; +using LocalPost; +using LocalPost.KafkaConsumer; +using LocalPost.KafkaConsumer.DependencyInjection; + +var builder = Host.CreateApplicationBuilder(args); + +builder.Services.Configure(options => +{ + options.ServicesStartConcurrently = true; + options.ServicesStopConcurrently = true; +}); + +builder.Services + .AddScoped() + .AddKafkaConsumers(kafka => + { + kafka.Defaults + .Bind(builder.Configuration.GetSection("Kafka")) + .ValidateDataAnnotations(); + kafka.AddConsumer("example-consumer-group", + HandlerStack.From() + .Scoped() + .UseKafkaPayload() + .Deserialize(context => JsonSerializer.Deserialize(context.Payload)!) + .Trace() + // .Acknowledge() + .LogExceptions() + ) + .Bind(builder.Configuration.GetSection("Kafka:Consumer")) + .Configure(options => + { + options.ClientConfig.AutoOffsetReset = AutoOffsetReset.Earliest; + // options.ClientConfig.EnableAutoCommit = false; // DryRun + // options.ClientConfig.EnableAutoOffsetStore = false; // Manually acknowledge every message + }) + .ValidateDataAnnotations(); + }); + +await builder.Build().RunAsync(); + + +[UsedImplicitly] +public record WeatherForecast(int TemperatureC, int TemperatureF, string Summary); + +internal sealed class MessageHandler : IHandler +{ + public ValueTask InvokeAsync(WeatherForecast payload, CancellationToken ct) + { + Console.WriteLine(payload); + return ValueTask.CompletedTask; + } +} diff --git a/examples/KafkaConsumerApp/Properties/launchSettings.json b/examples/KafkaConsumerApp/Properties/launchSettings.json new file mode 100644 index 0000000..0f2fe16 --- /dev/null +++ b/examples/KafkaConsumerApp/Properties/launchSettings.json @@ -0,0 +1,11 @@ +{ + "profiles": { + "KafkaConsumerApp": { + "commandName": "Project", + "dotnetRunMessages": true, + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} diff --git a/examples/KafkaConsumerApp/README.md b/examples/KafkaConsumerApp/README.md new file mode 100644 index 0000000..3a9614d --- /dev/null +++ b/examples/KafkaConsumerApp/README.md @@ -0,0 +1,3 @@ +# Kafka Consumer + +See https://github.com/confluentinc/confluent-kafka-dotnet/tree/master/examples/Protobuf diff --git a/examples/KafkaConsumerApp/appsettings.Development.json b/examples/KafkaConsumerApp/appsettings.Development.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/examples/KafkaConsumerApp/appsettings.Development.json @@ -0,0 +1 @@ +{} diff --git a/examples/KafkaConsumerApp/appsettings.json b/examples/KafkaConsumerApp/appsettings.json new file mode 100644 index 0000000..c3e961b --- /dev/null +++ b/examples/KafkaConsumerApp/appsettings.json @@ -0,0 +1,14 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "Kafka": { + "BootstrapServers": "127.0.0.1:19092", + "Consumer": { + "Topic": "weather-forecasts" + } + } +} diff --git a/examples/SqsConsumerApp/Program.cs b/examples/SqsConsumerApp/Program.cs new file mode 100644 index 0000000..4aae3c1 --- /dev/null +++ b/examples/SqsConsumerApp/Program.cs @@ -0,0 +1,103 @@ +using Amazon.SQS; +using JetBrains.Annotations; +using LocalPost; +using LocalPost.SqsConsumer; +using LocalPost.SqsConsumer.DependencyInjection; +using OpenTelemetry; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; +using Serilog; +using Serilog.Sinks.FingersCrossed; + +var builder = Host.CreateApplicationBuilder(args); + +builder.Services + .AddSerilog() // See https://nblumhardt.com/2024/04/serilog-net8-0-minimal/#hooking-up-aspnet-core-and-iloggert + .AddDefaultAWSOptions(builder.Configuration.GetAWSOptions()) + .AddAWSService(); + +builder.Services.Configure(options => +{ + options.ServicesStartConcurrently = true; + options.ServicesStopConcurrently = true; +}); + +#region OpenTelemetry + +// See also: https://learn.microsoft.com/en-us/dotnet/core/diagnostics/observability-otlp-example + +// To use full potential of Serilog, it's better to use Serilog.Sinks.OpenTelemetry, +// see https://github.com/Blind-Striker/dotnet-otel-aspire-localstack-demo as an example +// builder.Logging.AddOpenTelemetry(logging => +// { +// logging.IncludeFormattedMessage = true; +// logging.IncludeScopes = true; +// }); + +builder.Services.AddOpenTelemetry() + .WithMetrics(metrics => metrics + .AddAWSInstrumentation()) + .WithTracing(tracing => tracing + .AddSource("LocalPost.*") + .AddAWSInstrumentation()) + .UseOtlpExporter(); + +#endregion + +builder.Services + .AddScoped() + .AddSqsConsumers(sqs => + { + sqs.Defaults.Configure(options => options.MaxNumberOfMessages = 1); + sqs.AddConsumer("weather-forecasts", // Also acts as a queue name + HandlerStack.From() + .Scoped() + .UseSqsPayload() + .DeserializeJson() + .Trace() + .Acknowledge() // Do not include DeleteMessage call in the OpenTelemetry root span (transaction) + .LogFingersCrossed() + .LogExceptions() + ); + }); + +await builder.Build().RunAsync(); + + +[UsedImplicitly] +public record WeatherForecast(int TemperatureC, int TemperatureF, string Summary); + +public class MessageHandler : IHandler +{ + public async ValueTask InvokeAsync(WeatherForecast payload, CancellationToken ct) + { + await Task.Delay(1_000, ct); + Console.WriteLine(payload); + + // To show the failure handling + if (payload.TemperatureC > 35) + throw new InvalidOperationException("Too hot"); + } +} + +public static class HandlerStackEx +{ + public static HandlerManagerFactory LogFingersCrossed(this HandlerManagerFactory hmf) => + hmf.TouchHandler(next => async (context, ct) => + { + using var logBuffer = LogBuffer.BeginScope(); + try + { + await next(context, ct); + } + catch (OperationCanceledException e) when (e.CancellationToken == ct) + { + throw; // Do not treat cancellation as an error + } + catch (Exception) + { + logBuffer.Flush(); + throw; + } + }); +} diff --git a/examples/SqsConsumerApp/Properties/launchSettings.json b/examples/SqsConsumerApp/Properties/launchSettings.json new file mode 100644 index 0000000..4328853 --- /dev/null +++ b/examples/SqsConsumerApp/Properties/launchSettings.json @@ -0,0 +1,16 @@ +{ + "profiles": { + "SqsConsumerApp": { + "commandName": "Project", + "dotnetRunMessages": true, + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development", + "AWS_ACCESS_KEY_ID": "test", + "AWS_SECRET_ACCESS_KEY": "test", + "OTEL_SERVICE_NAME": "SampleSqsConsumer", + "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc", + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:18889" + } + } + } +} diff --git a/examples/SqsConsumerApp/README.md b/examples/SqsConsumerApp/README.md new file mode 100644 index 0000000..6261c32 --- /dev/null +++ b/examples/SqsConsumerApp/README.md @@ -0,0 +1,32 @@ +# SQS Consumer Sample App + +## Setup + +### Local infrastructure + +`docker compose up -d` to spin up the localstack & Aspire containers + +### SQS queue + +```shell +aws --endpoint-url=http://localhost:4566 --region=us-east-1 --no-sign-request \ + sqs create-queue --queue-name "weather-forecasts" +``` + +To get the queue URL: + +```shell +aws --endpoint-url=http://localhost:4566 --region=us-east-1 --no-sign-request \ + sqs get-queue-url --queue-name "weather-forecasts" --query "QueueUrl" +``` + +## Run + +To see that the consumer is working, you can send a message to the queue using the AWS CLI: + +```shell +aws --endpoint-url=http://localhost:4566 --region=us-east-1 --no-sign-request \ + sqs send-message \ + --queue-url "http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/weather-forecasts" \ + --message-body '{"TemperatureC": 25, "TemperatureF": 77, "Summary": "not hot, not cold, perfect"}' +``` diff --git a/examples/SqsConsumerApp/SqsConsumerApp.csproj b/examples/SqsConsumerApp/SqsConsumerApp.csproj new file mode 100644 index 0000000..6105d44 --- /dev/null +++ b/examples/SqsConsumerApp/SqsConsumerApp.csproj @@ -0,0 +1,37 @@ + + + + net8 + + CA1050 + + + + + + + + + + + + + + + + + + + + + + + + + + + appsettings.json + + + + diff --git a/examples/SqsConsumerApp/appsettings.Development.json b/examples/SqsConsumerApp/appsettings.Development.json new file mode 100644 index 0000000..ef8e3e4 --- /dev/null +++ b/examples/SqsConsumerApp/appsettings.Development.json @@ -0,0 +1,5 @@ +{ + "AWS": { + "ServiceURL": "http://127.0.0.1:4566" + } +} diff --git a/examples/SqsConsumerApp/appsettings.json b/examples/SqsConsumerApp/appsettings.json new file mode 100644 index 0000000..b2dcdb6 --- /dev/null +++ b/examples/SqsConsumerApp/appsettings.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/localstack/init/ready.d/sqs.sh b/localstack/init/ready.d/sqs.sh new file mode 100644 index 0000000..d47ca0d --- /dev/null +++ b/localstack/init/ready.d/sqs.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Enable debug +#set -x + +awslocal sqs create-queue --queue-name lp-test +QUEUE_URL=$(awslocal sqs get-queue-url --queue-name lp-test --query 'QueueUrl' --output text) + +awslocal sqs send-message \ + --queue-url "$QUEUE_URL" \ + --message-body '{"TemperatureC": 25, "TemperatureF": 77, "Summary": "not hot, not cold, perfect"}' diff --git a/samples/SampleWebApp/Program.cs b/samples/SampleWebApp/Program.cs deleted file mode 100644 index 2b58949..0000000 --- a/samples/SampleWebApp/Program.cs +++ /dev/null @@ -1,51 +0,0 @@ -using Amazon.SimpleNotificationService; -using Amazon.SQS; -using LocalPost.SnsPublisher.DependencyInjection; -using LocalPost.DependencyInjection; -using SampleWebApp; -using LocalPost.SqsConsumer.DependencyInjection; - -var builder = WebApplication.CreateBuilder(args); - - - -// A background queue with an inline handler -builder.Services.AddBackgroundQueue(_ => async (w, ct) => -{ - await Task.Delay(TimeSpan.FromSeconds(2), ct); - Console.WriteLine(w.Summary); -}); - - - -// An async Amazon SNS sender, buffers messages and sends them in batches in the background -builder.Services.AddAWSService(); -builder.Services.AddAmazonSnsBatchPublisher(); - - - -// An Amazon SQS consumer -builder.Services.AddAWSService(); -builder.Services.AddAmazonSqsMinimalConsumer("test", async (context, ct) => -{ - await Task.Delay(1_000, ct); - Console.WriteLine(context.Body); -}); - - - -builder.Services.AddControllers(); -// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle -builder.Services.AddEndpointsApiExplorer(); -builder.Services.AddSwaggerGen(); - -var app = builder.Build(); -if (app.Environment.IsDevelopment()) -{ - app.UseSwagger(); - app.UseSwaggerUI(); -} -app.UseHttpsRedirection(); -app.UseAuthorization(); -app.MapControllers(); -app.Run(); diff --git a/samples/SampleWebApp/SampleWebApp.csproj b/samples/SampleWebApp/SampleWebApp.csproj deleted file mode 100644 index 06fa56d..0000000 --- a/samples/SampleWebApp/SampleWebApp.csproj +++ /dev/null @@ -1,20 +0,0 @@ - - - - net7 - enable - - - - - - - - - - - - - - - diff --git a/sonar-scan.sh b/sonar-scan.sh index 3252ee4..7296c1f 100755 --- a/sonar-scan.sh +++ b/sonar-scan.sh @@ -1,9 +1,6 @@ #!/usr/bin/env bash -# Print a command before actually executing it -set -x -# Break the script if one of the command fails (returns non-zero status code) -set -e +set -o xtrace,errexit # $SONAR_TOKEN must be defined # $GitVersion_FullSemVer can be used to specify the current version (see GitVersion) @@ -15,9 +12,10 @@ fi dotnet build-server shutdown dotnet sonarscanner begin \ - /d:sonar.host.url="https://sonarcloud.io" /d:sonar.login="$SONAR_TOKEN" \ - /o:"alexeyshockov" /k:"alexeyshockov_LocalPost" $VERSION \ + /d:sonar.host.url="https://sonarcloud.io" /d:sonar.token="$SONAR_TOKEN" \ + /o:"alexeyshockov" /k:"alexeyshockov_LocalPost.NET" "$VERSION" \ /d:sonar.dotnet.excludeTestProjects=true \ + /d:sonar.coverage.exclusions="**/examples/**" \ /d:sonar.cs.opencover.reportsPaths="tests/*/TestResults/*/coverage.opencover.xml" \ /d:sonar.cs.vstest.reportsPaths="tests/*/TestResults/*.trx" @@ -25,4 +23,4 @@ dotnet sonarscanner begin \ dotnet build dotnet test --no-build --collect:"XPlat Code Coverage" --settings coverlet.runsettings --logger=trx -dotnet sonarscanner end /d:sonar.login="$SONAR_TOKEN" +dotnet sonarscanner end /d:sonar.token="$SONAR_TOKEN" diff --git a/src/LocalPost.SnsPublisher/LocalPost.SnsPublisher.csproj b/src/Directory.Build.props similarity index 50% rename from src/LocalPost.SnsPublisher/LocalPost.SnsPublisher.csproj rename to src/Directory.Build.props index bf2064c..6320993 100644 --- a/src/LocalPost.SnsPublisher/LocalPost.SnsPublisher.csproj +++ b/src/Directory.Build.props @@ -1,28 +1,23 @@ - + - netstandard2.0 + 13 + enable + enable + true + true false - LocalPost.SnsPublisher - background;task;queue;amazon;sns;aws - Local (in-process) background queue for sending to Amazon SNS. Alexey Shokov - - README.md + https://github.com/alexeyshockov/LocalPost.NET/releases/tag/v$(Version) MIT - https://github.com/alexeyshockov/LocalPost + https://github.com/alexeyshockov/LocalPost.NET git true - - - - - true @@ -40,17 +35,11 @@ - - - - - - - + + <_Parameter1>$(MSBuildProjectName).Tests + + + <_Parameter1>DynamicProxyGenAssembly2 + - - - - - diff --git a/src/LocalPost.KafkaConsumer/Client.cs b/src/LocalPost.KafkaConsumer/Client.cs new file mode 100644 index 0000000..f31938e --- /dev/null +++ b/src/LocalPost.KafkaConsumer/Client.cs @@ -0,0 +1,98 @@ +using System.Collections; +using Confluent.Kafka; + +namespace LocalPost.KafkaConsumer; + +internal sealed class ClientFactory(ILogger logger, ConsumerOptions settings) +{ + public async Task Create(CancellationToken ct) + { + return new Clients(await Task.WhenAll(Enumerable + .Range(0, settings.Consumers) + .Select(_ => Task.Run(CreateClient, ct)) + ).ConfigureAwait(false)); + + Client CreateClient() + { + var consumer = new ConsumerBuilder(settings.ClientConfig) + .SetErrorHandler((_, e) => logger.LogError("{Error}", e)) + .SetLogHandler((_, m) => logger.LogDebug(m.Message)) + .Build(); + consumer.Subscribe(settings.Topics); + return new Client(logger, consumer, settings.ClientConfig); + } + } +} + +internal sealed class Clients(Client[] clients) : IReadOnlyCollection +{ + public Task Close(CancellationToken ct) => Task.WhenAll(clients.Select(client => Task.Run(client.Close, ct))); + + public IEnumerator GetEnumerator() => ((IEnumerable)clients).GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => clients.GetEnumerator(); + + public int Count => clients.Length; +} + +internal sealed class Client +{ + private readonly ILogger _logger; + + public Client(ILogger logger, IConsumer consumer, ConsumerConfig config) + { + _logger = logger; + Consumer = consumer; + Config = config; + var server = config.BootstrapServers.Split(',')[0].Split(':'); + ServerAddress = server[0]; + if (server.Length > 1) + ServerPort = int.Parse(server[1]); + } + + public ConsumeResult Consume(CancellationToken ct) + { + while (true) + { + try + { + var result = Consumer.Consume(ct); + + if (result is not null && result.IsPartitionEOF) + _logger.LogInformation("End of {Partition} on {Topic}", + result.TopicPartition.Partition, result.TopicPartition.Topic); + else if (result?.Message is not null) + return result; + else + _logger.LogWarning("Kafka consumer empty receive"); + } + // catch (ConsumeException e) + catch (KafkaException e) when (!e.Error.IsFatal) + { + _logger.LogCritical(e, "Kafka consumer (retryable) error: {Reason}", e.Error.Reason); + // Just continue receiving + + // "generally, the producer should recover from all errors, except where marked fatal" as per + // https://github.com/confluentinc/confluent-kafka-dotnet/issues/1213#issuecomment-599772818, so + // just continue polling + } + } + } + + public void Close() + { + try + { + Consumer.Close(); + } + finally + { + Consumer.Dispose(); + } + } + + public IConsumer Consumer { get; } + public ConsumerConfig Config { get; } + public string ServerAddress { get; } + public int ServerPort { get; } = 9092; +} diff --git a/src/LocalPost.KafkaConsumer/ConsumeContext.cs b/src/LocalPost.KafkaConsumer/ConsumeContext.cs new file mode 100644 index 0000000..6674045 --- /dev/null +++ b/src/LocalPost.KafkaConsumer/ConsumeContext.cs @@ -0,0 +1,52 @@ +using Confluent.Kafka; + +namespace LocalPost.KafkaConsumer; + +[PublicAPI] +public readonly record struct ConsumeContext +{ + // librdkafka docs: + // When consumer restarts this is where it will start consuming from. + // The committed offset should be last_message_offset+1. + // See https://github.com/confluentinc/librdkafka/wiki/Consumer-offset-management#terminology +// internal readonly TopicPartitionOffset NextOffset; + + internal readonly Client Client; + internal readonly ConsumeResult ConsumeResult; + public readonly T Payload; + + internal ConsumeContext(Client client, ConsumeResult consumeResult, T payload) + { + Client = client; + ConsumeResult = consumeResult; + Payload = payload; + } + + public void Deconstruct(out T payload, out IReadOnlyList headers) + { + payload = Payload; + headers = Headers; + } + + public Offset NextOffset => ConsumeResult.Offset + 1; + + public Message Message => ConsumeResult.Message; + + public string Topic => ConsumeResult.Topic; + + public IReadOnlyList Headers => Message.Headers.BackingList; + + public ConsumeContext Transform(TOut payload) => new(Client, ConsumeResult, payload); + + public ConsumeContext Transform(Func, TOut> transform) => Transform(transform(this)); + + public async Task> Transform(Func, Task> transform) => + Transform(await transform(this).ConfigureAwait(false)); + + public static implicit operator T(ConsumeContext context) => context.Payload; + + public void StoreOffset() => Client.Consumer.StoreOffset(ConsumeResult); + + // To be consistent across different message brokers + public void Acknowledge() => StoreOffset(); +} diff --git a/src/LocalPost.KafkaConsumer/Consumer.cs b/src/LocalPost.KafkaConsumer/Consumer.cs new file mode 100644 index 0000000..a8d4737 --- /dev/null +++ b/src/LocalPost.KafkaConsumer/Consumer.cs @@ -0,0 +1,126 @@ +using Confluent.Kafka; +using LocalPost.DependencyInjection; +using LocalPost.Flow; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; + +namespace LocalPost.KafkaConsumer; + +internal sealed class Consumer(string name, ILogger logger, + ClientFactory clientFactory, IHandlerManager> hm) + : IHostedService, IHealthAwareService, IDisposable +{ + private Clients _clients = new([]); + + private CancellationTokenSource? _execTokenSource; + private Task? _exec; + private Exception? _execException; + private string? _execExceptionDescription; + + private CancellationToken _completionToken = CancellationToken.None; + + private HealthCheckResult Ready => (_execTokenSource, _execution: _exec, _execException) switch + { + (null, _, _) => HealthCheckResult.Unhealthy("Not started"), + (_, { IsCompleted: true }, _) => HealthCheckResult.Unhealthy("Stopped"), + (not null, null, _) => HealthCheckResult.Degraded("Starting"), + (not null, not null, null) => HealthCheckResult.Healthy("Running"), + (_, _, not null) => HealthCheckResult.Unhealthy(_execExceptionDescription, _execException), + }; + + public IHealthCheck ReadinessCheck => HealthChecks.From(() => Ready); + + private async Task RunConsumerAsync(Client client, Handler> handler, CancellationToken execToken) + { + // (Optionally) wait for app start + + try + { + while (!execToken.IsCancellationRequested) + { + var result = client.Consume(execToken); + var context = new ConsumeContext(client, result, result.Message.Value); + await handler(context, CancellationToken.None).ConfigureAwait(false); + } + } + catch (OperationCanceledException e) when (e.CancellationToken == execToken) + { + // logger.LogInformation("Kafka consumer shutdown"); + } + catch (KafkaException e) + { + logger.LogCritical(e, "Kafka consumer error: {Reason} (see {HelpLink})", e.Error.Reason, e.HelpLink); + (_execException, _execExceptionDescription) = (e, "Kafka consumer failed"); + } + catch (Exception e) + { + logger.LogCritical(e, "Kafka message handler error"); + // TODO Include headers or the partition key in check result's data + (_execException, _execExceptionDescription) = (e, "Message handler failed"); + } + finally + { + CancelExecution(); // Stop other consumers too + } + } + + public async Task StartAsync(CancellationToken ct) + { + if (_execTokenSource is not null) + throw new InvalidOperationException("Service is already started"); + + var execTokenSource = _execTokenSource = new CancellationTokenSource(); + + logger.LogInformation("Starting Kafka consumer..."); + var clients = _clients = await clientFactory.Create(ct).ConfigureAwait(false); + logger.LogInformation("Kafka consumer started"); + + logger.LogDebug("Invoking the event handler..."); + var handler = await hm.Start(ct).ConfigureAwait(false); + logger.LogDebug("Event handler started"); + + _exec = ObserveExecution(); + return; + + async Task ObserveExecution() + { + try + { + var executions = clients.Select(client => + Task.Run(() => RunConsumerAsync(client, handler, execTokenSource.Token), ct) + ).ToArray(); + await (executions.Length == 1 ? executions[0] : Task.WhenAll(executions)).ConfigureAwait(false); + + await hm.Stop(_execException, _completionToken).ConfigureAwait(false); + } + finally + { + // Can happen before the service shutdown, in case of an error + await _clients.Close(_completionToken).ConfigureAwait(false); + logger.LogInformation("Kafka consumer stopped"); + } + } + } + + // await _execTokenSource.CancelAsync(); // .NET 8+ + private void CancelExecution() => _execTokenSource?.Cancel(); + + public async Task StopAsync(CancellationToken forceShutdownToken) + { + if (_execTokenSource is null) + throw new InvalidOperationException("Service has not been started"); + + logger.LogInformation("Shutting down Kafka consumer..."); + + _completionToken = forceShutdownToken; + CancelExecution(); + if (_exec is not null) + await _exec.ConfigureAwait(false); + } + + public void Dispose() + { + _execTokenSource?.Dispose(); + _exec?.Dispose(); + } +} diff --git a/src/LocalPost.KafkaConsumer/DependencyInjection/HealthChecksBuilderEx.cs b/src/LocalPost.KafkaConsumer/DependencyInjection/HealthChecksBuilderEx.cs new file mode 100644 index 0000000..a4fb1c3 --- /dev/null +++ b/src/LocalPost.KafkaConsumer/DependencyInjection/HealthChecksBuilderEx.cs @@ -0,0 +1,22 @@ +using LocalPost.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace LocalPost.KafkaConsumer.DependencyInjection; + +[PublicAPI] +public static class HealthChecksBuilderEx +{ + public static IHealthChecksBuilder AddKafkaConsumer(this IHealthChecksBuilder builder, + string name, HealthStatus? failureStatus = null, IEnumerable? tags = null) => + builder.Add(HealthChecks.Readiness(name, failureStatus, tags)); + + public static IHealthChecksBuilder AddKafkaConsumers(this IHealthChecksBuilder builder, + HealthStatus? failureStatus = null, IEnumerable? tags = null) + { + foreach (var name in builder.Services.GetKeysFor().OfType()) + AddKafkaConsumer(builder, name, failureStatus, tags); + + return builder; + } +} diff --git a/src/LocalPost.KafkaConsumer/DependencyInjection/KafkaBuilder.cs b/src/LocalPost.KafkaConsumer/DependencyInjection/KafkaBuilder.cs new file mode 100644 index 0000000..cd5e200 --- /dev/null +++ b/src/LocalPost.KafkaConsumer/DependencyInjection/KafkaBuilder.cs @@ -0,0 +1,58 @@ +using Confluent.Kafka; +using LocalPost.DependencyInjection; +using LocalPost.Flow; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace LocalPost.KafkaConsumer.DependencyInjection; + +[PublicAPI] +public sealed class KafkaBuilder(IServiceCollection services) +{ + public OptionsBuilder Defaults { get; } = services.AddOptions(); + + /// + /// Add a Kafka consumer with a custom message handler. + /// + /// Message handler factory. + /// Consumer options builder. + public OptionsBuilder AddConsumer(HandlerManagerFactory> hmf) => + AddConsumer(Options.DefaultName, hmf); + + /// + /// Add a Kafka consumer with a custom message handler. + /// + /// Consumer name (should be unique in the application). Also, the default group ID. + /// Message handler factory. + /// Consumer options builder. + public OptionsBuilder AddConsumer(string name, HandlerManagerFactory> hmf) + { + var added = services.TryAddKeyedSingleton(name, (provider, _) => + { + var clientFactory = new ClientFactory( + provider.GetLoggerFor(), + provider.GetOptions(name) + ); + + return new Consumer(name, + provider.GetLoggerFor(), + clientFactory, + hmf(provider) + ); + }); + + if (!added) + throw new ArgumentException("Consumer is already registered", nameof(name)); + + services.AddHostedService(provider => provider.GetRequiredKeyedService(name)); + + return OptionsFor(name).Configure>((co, defaults) => + { + co.EnrichFrom(defaults.Value); + if (!string.IsNullOrEmpty(name)) + co.ClientConfig.GroupId = name; + }); + } + + public OptionsBuilder OptionsFor(string name) => services.AddOptions(name); +} diff --git a/src/LocalPost.KafkaConsumer/DependencyInjection/ServiceCollectionEx.cs b/src/LocalPost.KafkaConsumer/DependencyInjection/ServiceCollectionEx.cs new file mode 100644 index 0000000..67d65f4 --- /dev/null +++ b/src/LocalPost.KafkaConsumer/DependencyInjection/ServiceCollectionEx.cs @@ -0,0 +1,14 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace LocalPost.KafkaConsumer.DependencyInjection; + +[PublicAPI] +public static class ServiceCollectionEx +{ + public static IServiceCollection AddKafkaConsumers(this IServiceCollection services, Action configure) + { + configure(new KafkaBuilder(services)); + + return services; + } +} diff --git a/src/LocalPost.KafkaConsumer/Exceptions.cs b/src/LocalPost.KafkaConsumer/Exceptions.cs new file mode 100644 index 0000000..782d421 --- /dev/null +++ b/src/LocalPost.KafkaConsumer/Exceptions.cs @@ -0,0 +1,10 @@ +using Confluent.Kafka; + +namespace LocalPost.KafkaConsumer; + +internal static class Exceptions +{ + public static bool IsTransient(this ConsumeException exception) => + // See https://github.com/confluentinc/confluent-kafka-dotnet/issues/1424#issuecomment-705749252 + exception.Error.Code is ErrorCode.Local_KeyDeserialization or ErrorCode.Local_ValueDeserialization; +} diff --git a/src/LocalPost.KafkaConsumer/HandlerStackEx.cs b/src/LocalPost.KafkaConsumer/HandlerStackEx.cs new file mode 100644 index 0000000..8d77325 --- /dev/null +++ b/src/LocalPost.KafkaConsumer/HandlerStackEx.cs @@ -0,0 +1,150 @@ +using Confluent.Kafka; + +namespace LocalPost.KafkaConsumer; + +using MessageHmf = HandlerManagerFactory>; +using MessagesHmf = HandlerManagerFactory>>; + +[PublicAPI] +public static class HandlerStackEx +{ + public static HandlerManagerFactory> UseKafkaPayload(this HandlerManagerFactory hmf) => + hmf.MapHandler, T>(next => async (context, ct) => + await next(context.Payload, ct).ConfigureAwait(false)); + + public static HandlerManagerFactory>> UseKafkaPayload( + this HandlerManagerFactory> hmf) => + hmf.MapHandler>, IReadOnlyCollection>(next => async (batch, ct) => + await next(batch.Select(context => context.Payload).ToArray(), ct).ConfigureAwait(false)); + + public static HandlerManagerFactory> Trace( + this HandlerManagerFactory> hmf) => + hmf.TouchHandler(next => async (context, ct) => + { + using var activity = Tracing.StartProcessing(context); + try + { + await next(context, ct).ConfigureAwait(false); + activity?.Success(); + } + catch (Exception e) + { + activity?.Error(e); + throw; + } + }); + + public static HandlerManagerFactory>> Trace( + this HandlerManagerFactory>> hmf) => + hmf.TouchHandler(next => async (batch, ct) => + { + using var activity = Tracing.StartProcessing(batch); + try + { + await next(batch, ct).ConfigureAwait(false); + activity?.Success(); + } + catch (Exception e) + { + activity?.Error(e); + throw; + } + }); + + /// + /// Manually acknowledge every message (store offset). + /// + /// Works only when EnableAutoOffsetStore is false! + /// + /// Message handler factory. + /// Message type. + /// Wrapped handler factory. + public static HandlerManagerFactory> Acknowledge( + this HandlerManagerFactory> hmf) => + hmf.TouchHandler(next => async (context, ct) => + { + await next(context, ct).ConfigureAwait(false); + context.StoreOffset(); + }); + + /// + /// Manually acknowledge every message (store offset). + /// + /// Works only when EnableAutoOffsetStore is false! + /// + /// Message handler factory. + /// Message type. + /// Wrapped handler factory. + public static HandlerManagerFactory>> Acknowledge( + this HandlerManagerFactory>> hmf) => + hmf.TouchHandler(next => async (batch, ct) => + { + await next(batch, ct).ConfigureAwait(false); + foreach (var context in batch) + context.StoreOffset(); + }); + + private static Func, Task> AsyncDeserializer(IAsyncDeserializer deserializer) => + context => deserializer.DeserializeAsync(context.Payload, false, new SerializationContext( + MessageComponentType.Value, context.Topic, context.Message.Headers)); + + private static Func, T> Deserializer(IDeserializer deserializer) => + context => deserializer.Deserialize(context.Payload, false, new SerializationContext( + MessageComponentType.Value, context.Topic, context.Message.Headers)); + + #region Deserialize() + + public static MessageHmf Deserialize(this HandlerManagerFactory> hmf, + Func, T> deserialize) => + hmf.MapHandler, ConsumeContext>(next => async (context, ct) => + await next(context.Transform(deserialize), ct).ConfigureAwait(false)); + + public static MessageHmf Deserialize(this HandlerManagerFactory> hmf, + Func, Task> deserialize) => + hmf.MapHandler, ConsumeContext>(next => async (context, ct) => + await next(await context.Transform(deserialize).ConfigureAwait(false), ct).ConfigureAwait(false)); + + public static MessageHmf Deserialize(this HandlerManagerFactory> hmf, + IAsyncDeserializer deserializer) => hmf.Deserialize(AsyncDeserializer(deserializer)); + + public static MessageHmf Deserialize(this HandlerManagerFactory> hmf, + IDeserializer deserializer) => hmf.Deserialize(Deserializer(deserializer)); + + + + public static MessagesHmf Deserialize( + this HandlerManagerFactory>> hmf, + Func, T> deserialize) => + hmf.MapHandler>, IReadOnlyCollection>>(next => + async (batch, ct) => + { + var modBatch = batch.Select(context => context.Transform(deserialize)).ToArray(); + await next(modBatch, ct).ConfigureAwait(false); + }); + + public static MessagesHmf Deserialize( + this HandlerManagerFactory>> hmf, + Func, Task> deserialize) => + hmf.MapHandler>, IReadOnlyCollection>>(next => + async (batch, ct) => + { + var modifications = batch.Select(context => context.Transform(deserialize)); + // Task.WhenAll() preserves the order + var modBatch = await Task.WhenAll(modifications).ConfigureAwait(false); + await next(modBatch, ct).ConfigureAwait(false); + }); + + public static MessagesHmf Deserialize( + this HandlerManagerFactory>> hmf, + IAsyncDeserializer deserializer) => hmf.Deserialize(AsyncDeserializer(deserializer)); + + public static MessagesHmf Deserialize( + this HandlerManagerFactory>> hmf, + IDeserializer deserializer) => hmf.Deserialize(Deserializer(deserializer)); + + #endregion + + // public static HandlerFactory> DeserializeJson( + // this HandlerFactory> hf, JsonSerializerOptions? options = null) => + // hf.Deserialize(context => JsonSerializer.Deserialize(context.Payload, options)!); +} diff --git a/src/LocalPost.KafkaConsumer/LocalPost.KafkaConsumer.csproj b/src/LocalPost.KafkaConsumer/LocalPost.KafkaConsumer.csproj new file mode 100644 index 0000000..9e4add7 --- /dev/null +++ b/src/LocalPost.KafkaConsumer/LocalPost.KafkaConsumer.csproj @@ -0,0 +1,34 @@ + + + + net6;net8 + + LocalPost.KafkaConsumer + + Opinionated Kafka consumer library, build to be simple, but yet flexible. + background;task;queue;kafka + + README.md + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/LocalPost.KafkaConsumer/Options.cs b/src/LocalPost.KafkaConsumer/Options.cs new file mode 100644 index 0000000..368d500 --- /dev/null +++ b/src/LocalPost.KafkaConsumer/Options.cs @@ -0,0 +1,25 @@ +using System.ComponentModel.DataAnnotations; +using Confluent.Kafka; + +namespace LocalPost.KafkaConsumer; + +[UsedImplicitly] +public sealed record ConsumerOptions +{ + public ConsumerConfig ClientConfig { get; set; } = new(); + // { + // EnableAutoOffsetStore = false // Store offsets manually, see Acknowledge middleware + // }; + + [MinLength(1)] + public ISet Topics { get; set; } = new HashSet(); + + [Range(1, ushort.MaxValue)] + public ushort Consumers { get; set; } = 1; + + internal void EnrichFrom(Config config) + { + foreach (var kv in config) + ClientConfig.Set(kv.Key, kv.Value); + } +} diff --git a/src/LocalPost.KafkaConsumer/README.md b/src/LocalPost.KafkaConsumer/README.md new file mode 100644 index 0000000..84e199c --- /dev/null +++ b/src/LocalPost.KafkaConsumer/README.md @@ -0,0 +1,25 @@ +# LocalPost Kafka Consumer + +## librdkafka's background prefetching + +The Kafka client automatically prefetches messages in the background. This is done by the background thread that is +started when the client is created. The background thread will fetch messages from the broker and enqueue them on the +internal queue, so `Consume()` calls will return faster. + +Because of this behavior, there is no need to maintain our own in memory queue (channel). + +## Concurrent processing + +A Kafka consumer is designed to handle messages _from one partition_ sequentially, as it commits the offset of the last +processed message. + +One of the common ways to speed up things (increase throughput) is to have multiple partitions for a topic and multiple +parallel consumers. + +Another way is to batch process messages. + +## Message key ignorance + +Kafka's message key is used for almost one and only one purpose: to determine the partition for the message, when +publishing. And in almost all the cases this information is also available (serialized) in the message itself +(message value in Kafka terms). That's why we are ignoring the message key in this consumer. diff --git a/src/LocalPost.KafkaConsumer/Tracing.cs b/src/LocalPost.KafkaConsumer/Tracing.cs new file mode 100644 index 0000000..e67f1a3 --- /dev/null +++ b/src/LocalPost.KafkaConsumer/Tracing.cs @@ -0,0 +1,137 @@ +using System.Diagnostics; +using System.Reflection; +using System.Text; +using Confluent.Kafka; + +namespace LocalPost.KafkaConsumer; + +internal static class MessageUtils +{ + public static void ExtractTraceFieldFromHeaders(object? carrier, string fieldName, + out string? fieldValue, out IEnumerable? fieldValues) + { + fieldValues = null; + fieldValue = null; + if (carrier is not IEnumerable message) + return; + + var headerValue = message.FirstOrDefault(header => header.Key == fieldName)?.GetValueBytes(); + if (headerValue is not null) + fieldValue = Encoding.UTF8.GetString(headerValue); + } +} + +// See https://opentelemetry.io/docs/specs/semconv/messaging/kafka/ +internal static class KafkaActivityExtensions +{ + public static void AcceptDistributedTracingFrom(this Activity activity, Message message) + { + var propagator = DistributedContextPropagator.Current; + propagator.ExtractTraceIdAndState(message.Headers, MessageUtils.ExtractTraceFieldFromHeaders, + out var traceParent, out var traceState); + + if (string.IsNullOrEmpty(traceParent)) + return; + activity.SetParentId(traceParent!); + if (!string.IsNullOrEmpty(traceState)) + activity.TraceStateString = traceState; + + var baggage = propagator.ExtractBaggage(message.Headers, MessageUtils.ExtractTraceFieldFromHeaders); + if (baggage is null) + return; + foreach (var baggageItem in baggage) + activity.AddBaggage(baggageItem.Key, baggageItem.Value); + } + + public static Activity? SetDefaultTags(this Activity? activity, Client client) + { + // See https://opentelemetry.io/docs/specs/semconv/messaging/messaging-spans/#messaging-attributes + activity?.SetTag("messaging.system", "kafka"); + + // activity?.SetTag("messaging.kafka.consumer.group", context.ClientConfig.GroupId); + activity?.SetTag("messaging.consumer.group.name", client.Config.GroupId); + + // activity?.SetTag("messaging.client.id", "service_name"); + // activity?.SetTag("server.address", context.ClientConfig.BootstrapServers); + // activity?.SetTag("server.port", context.ClientConfig.BootstrapServers); + + return activity; + } + + public static Activity? SetTagsFor(this Activity? activity, ConsumeContext context) + { + // See https://github.com/open-telemetry/opentelemetry-specification/issues/2971#issuecomment-1324621326 + // activity?.SetTag("messaging.message.id", context.MessageId); + activity?.SetTag("messaging.destination.name", context.Topic); + activity?.SetTag("messaging.destination.partition.id", context.ConsumeResult.Partition.Value); + activity?.SetTag("messaging.kafka.message.offset", (long)context.ConsumeResult.Offset); + + activity?.SetTag("messaging.message.body.size", context.Message.Value.Length); + + // Skip, as we always ignore the key on consumption + // activity.SetTag("messaging.kafka.message.key", context.Message.Key); + + // TODO error.type + + return activity; + } + + public static Activity? SetTagsFor(this Activity? activity, IReadOnlyCollection> batch) + { + activity?.SetTag("messaging.batch.message_count", batch.Count); + if (batch.Count > 0) + activity?.SetTag("messaging.destination.name", batch.First().Topic); + + return activity; + } + + // public static Activity? SetTagsFor(this Activity? activity, IReadOnlyCollection> batch) => + // activity?.SetTag("messaging.batch.message_count", batch.Count); +} + +// Based on Semantic Conventions 1.30.0, see +// https://opentelemetry.io/docs/specs/semconv/messaging/messaging-spans/ +// Also Npgsql as an inspiration: +// - https://github.com/npgsql/npgsql/blob/main/src/Npgsql/NpgsqlActivitySource.cs +// - https://github.com/npgsql/npgsql/blob/main/src/Npgsql/NpgsqlCommand.cs +internal static class Tracing +{ + private static readonly ActivitySource Source; + + public static bool IsEnabled => Source.HasListeners(); + + static Tracing() + { + var assembly = Assembly.GetExecutingAssembly(); + var version = assembly.GetName().Version?.ToString() ?? "0.0.0"; + Source = new ActivitySource("LocalPost.KafkaConsumer", version); + } + + public static Activity? StartProcessing(IReadOnlyCollection> batch) + { + Debug.Assert(batch.Count > 0); + var activity = Source.StartActivity($"process {batch.First().Topic}", ActivityKind.Consumer); + if (activity is not { IsAllDataRequested: true }) + return activity; + + activity.SetTag("messaging.operation.type", "process"); + activity.SetDefaultTags(batch.First().Client); + activity.SetTagsFor(batch); + + return activity; + } + + public static Activity? StartProcessing(ConsumeContext context) + { + var activity = Source.StartActivity($"process {context.Topic}", ActivityKind.Consumer); + if (activity is not { IsAllDataRequested: true }) + return activity; + + activity.SetTag("messaging.operation.type", "process"); + activity.SetDefaultTags(context.Client); + activity.SetTagsFor(context); + activity.AcceptDistributedTracingFrom(context.Message); + + return activity; + } +} diff --git a/src/LocalPost.KafkaConsumer/globalusings.cs b/src/LocalPost.KafkaConsumer/globalusings.cs new file mode 100644 index 0000000..2f865c2 --- /dev/null +++ b/src/LocalPost.KafkaConsumer/globalusings.cs @@ -0,0 +1,3 @@ +global using JetBrains.Annotations; +global using System.Diagnostics.CodeAnalysis; +global using Microsoft.Extensions.Logging; diff --git a/src/LocalPost.SnsPublisher/DependencyInjection/ServiceCollectionExtensions.cs b/src/LocalPost.SnsPublisher/DependencyInjection/ServiceCollectionExtensions.cs deleted file mode 100644 index f44a234..0000000 --- a/src/LocalPost.SnsPublisher/DependencyInjection/ServiceCollectionExtensions.cs +++ /dev/null @@ -1,27 +0,0 @@ -using Amazon.SimpleNotificationService.Model; -using LocalPost.DependencyInjection; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; - -namespace LocalPost.SnsPublisher.DependencyInjection; - -public static class ServiceCollectionExtensions -{ - public static OptionsBuilder AddAmazonSnsBatchPublisher(this IServiceCollection services) - { - services.TryAddSingleton(); - - return services - .AddAmazonSnsBatchPublisher(provider => provider.GetRequiredService().Send); - } - - public static OptionsBuilder AddAmazonSnsBatchPublisher(this IServiceCollection services, - Func> handlerFactory) - { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService()); - - return services.AddCustomBackgroundQueue(handlerFactory); - } -} diff --git a/src/LocalPost.SnsPublisher/PublishBatchRequestEntryExtensions.cs b/src/LocalPost.SnsPublisher/PublishBatchRequestEntryExtensions.cs deleted file mode 100644 index 4abb6b6..0000000 --- a/src/LocalPost.SnsPublisher/PublishBatchRequestEntryExtensions.cs +++ /dev/null @@ -1,10 +0,0 @@ -using System.Text; -using Amazon.SimpleNotificationService.Model; - -namespace LocalPost.SnsPublisher; - -internal static class PublishBatchRequestEntryExtensions -{ - // Include attributes in the calculation later?.. - public static int CalculateSize(this PublishBatchRequestEntry entry) => Encoding.UTF8.GetByteCount(entry.Message); -} diff --git a/src/LocalPost.SnsPublisher/Publisher.cs b/src/LocalPost.SnsPublisher/Publisher.cs deleted file mode 100644 index 6115d8d..0000000 --- a/src/LocalPost.SnsPublisher/Publisher.cs +++ /dev/null @@ -1,59 +0,0 @@ -using System.Threading.Channels; -using Amazon.SimpleNotificationService.Model; - -namespace LocalPost.SnsPublisher; - -public interface ISnsPublisher -{ - IBackgroundQueue ForTopic(string arn); -} - -internal sealed class Publisher : ISnsPublisher, IAsyncEnumerable, IDisposable -{ - private sealed class TopicPublishingQueue : IBackgroundQueue - { - private readonly Channel _batchEntries; - - public TopicPublishingQueue(string arn) - { - _batchEntries = Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false - }); - Results = _batchEntries.Reader.ReadAllAsync().Batch(() => new SnsBatchBuilder(arn)); - } - - public IAsyncEnumerable Results { get; } - - public ValueTask Enqueue(PublishBatchRequestEntry item, CancellationToken ct = default) - { - if (item.CalculateSize() > PublisherOptions.RequestMaxSize) - throw new ArgumentOutOfRangeException(nameof(item), "Message is too big"); - - return _batchEntries.Writer.WriteAsync(item, ct); - } - } - - private readonly Dictionary _channels = new(); - private readonly AsyncEnumerableMerger _combinedReader = new(true); - - private TopicPublishingQueue Create(string arn) - { - var q = _channels[arn] = new TopicPublishingQueue(arn); - _combinedReader.Add(q.Results); - - return q; - } - - public IBackgroundQueue ForTopic(string arn) => - _channels.TryGetValue(arn, out var queue) ? queue : Create(arn); - - public void Dispose() - { - _combinedReader.Dispose(); - } - - public IAsyncEnumerator GetAsyncEnumerator(CancellationToken ct = default) => - _combinedReader.GetAsyncEnumerator(ct); -} diff --git a/src/LocalPost.SnsPublisher/PublisherOptions.cs b/src/LocalPost.SnsPublisher/PublisherOptions.cs deleted file mode 100644 index dd16e71..0000000 --- a/src/LocalPost.SnsPublisher/PublisherOptions.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace LocalPost.SnsPublisher; - -public sealed record PublisherOptions -{ - // Same for Publish and PublishBatch - public const int RequestMaxSize = 262_144; - - public const int BatchMaxSize = 10; -} diff --git a/src/LocalPost.SnsPublisher/Sender.cs b/src/LocalPost.SnsPublisher/Sender.cs deleted file mode 100644 index da477cc..0000000 --- a/src/LocalPost.SnsPublisher/Sender.cs +++ /dev/null @@ -1,30 +0,0 @@ -using Amazon.SimpleNotificationService; -using Amazon.SimpleNotificationService.Model; -using Microsoft.Extensions.Logging; - -namespace LocalPost.SnsPublisher; - -/// -/// Default implementation -/// -internal sealed class Sender -{ - private readonly ILogger _logger; - private readonly IAmazonSimpleNotificationService _sns; - - public Sender(ILogger logger, IAmazonSimpleNotificationService sns) - { - _logger = logger; - _sns = sns; - } - - public async Task Send(PublishBatchRequest payload, CancellationToken ct) - { - _logger.LogTrace("Sending a batch of {Amount} publish request(s) to SNS...", payload.PublishBatchRequestEntries.Count); - var batchResponse = await _sns.PublishBatchAsync(payload, ct); - - if (batchResponse.Failed.Any()) - _logger.LogError("Batch entries failed: {FailedAmount} from {Amount}", - batchResponse.Failed.Count, payload.PublishBatchRequestEntries.Count); - } -} diff --git a/src/LocalPost.SnsPublisher/SnsBatchBuilder.cs b/src/LocalPost.SnsPublisher/SnsBatchBuilder.cs deleted file mode 100644 index fa3b2b5..0000000 --- a/src/LocalPost.SnsPublisher/SnsBatchBuilder.cs +++ /dev/null @@ -1,56 +0,0 @@ -using Amazon.SimpleNotificationService.Model; -using Nito.AsyncEx; - -namespace LocalPost.SnsPublisher; - -internal sealed class SnsBatchBuilder : IBatchBuilder -{ - private readonly CancellationTokenSource _timeWindow = new(TimeSpan.FromSeconds(1)); // TODO Configurable - private readonly CancellationTokenTaskSource _timeWindowTrigger; - private PublishBatchRequest? _batchRequest; - - public SnsBatchBuilder(string topicArn) - { - _batchRequest = new PublishBatchRequest - { - TopicArn = topicArn - }; - - _timeWindowTrigger = new CancellationTokenTaskSource(_timeWindow.Token); - } - - private PublishBatchRequest BatchRequest => _batchRequest ?? throw new ObjectDisposedException(nameof(SnsBatchBuilder)); - - public CancellationToken TimeWindow => _timeWindow.Token; - public Task TimeWindowTrigger => _timeWindowTrigger.Task; - public bool IsEmpty => BatchRequest.PublishBatchRequestEntries.Count == 0; - - private bool CanFit(PublishBatchRequestEntry entry) => - PublisherOptions.BatchMaxSize > BatchRequest.PublishBatchRequestEntries.Count - && - PublisherOptions.RequestMaxSize > BatchRequest.PublishBatchRequestEntries.Append(entry) - .Aggregate(0, (total, e) => total + e.CalculateSize()); - - public bool TryAdd(PublishBatchRequestEntry entry) - { - var canFit = CanFit(entry); - if (!canFit) - return false; - - if (string.IsNullOrEmpty(entry.Id)) - entry.Id = Guid.NewGuid().ToString(); - - BatchRequest.PublishBatchRequestEntries.Add(entry); - - return true; - } - - public PublishBatchRequest Build() => BatchRequest; - - public void Dispose() - { - _timeWindow.Dispose(); - _timeWindowTrigger.Dispose(); - _batchRequest = null; // Just make it unusable - } -} diff --git a/src/LocalPost.SqsConsumer/ConsumeContext.cs b/src/LocalPost.SqsConsumer/ConsumeContext.cs new file mode 100644 index 0000000..5dc8e58 --- /dev/null +++ b/src/LocalPost.SqsConsumer/ConsumeContext.cs @@ -0,0 +1,56 @@ +using Amazon.SQS.Model; + +namespace LocalPost.SqsConsumer; + +[PublicAPI] +public readonly record struct ConsumeContext +{ + internal readonly QueueClient Client; + internal readonly Message Message; + public readonly T Payload; + + public DateTimeOffset ReceivedAt { get; init; } = DateTimeOffset.Now; + + internal ConsumeContext(QueueClient client, Message message, T payload) + { + Client = client; + Payload = payload; + Message = message; + } + + public void Deconstruct(out T payload, out Message message) + { + payload = Payload; + message = Message; + } + + public string MessageId => Message.MessageId; + + public string ReceiptHandle => Message.ReceiptHandle; + + public IReadOnlyDictionary Attributes => Message.Attributes; + + public IReadOnlyDictionary MessageAttributes => Message.MessageAttributes; + + public bool IsStale => false; // TODO Check the visibility timeout + + public ConsumeContext Transform(TOut payload) => + new(Client, Message, payload) + { + ReceivedAt = ReceivedAt + }; + + public ConsumeContext Transform(Func, TOut> transform) => Transform(transform(this)); + + public async Task> Transform(Func, Task> transform) => + Transform(await transform(this).ConfigureAwait(false)); + + public static implicit operator T(ConsumeContext context) => context.Payload; + + public Task DeleteMessage(CancellationToken ct = default) => Client.DeleteMessage(this, ct); + + public Task Acknowledge(CancellationToken ct = default) => DeleteMessage(ct); + + // public Task ChangeMessageVisibility(TimeSpan visibilityTimeout, CancellationToken ct = default) => + // Client.ChangeMessageVisibility(this, visibilityTimeout, ct); +} diff --git a/src/LocalPost.SqsConsumer/Consumer.cs b/src/LocalPost.SqsConsumer/Consumer.cs new file mode 100644 index 0000000..7ee6752 --- /dev/null +++ b/src/LocalPost.SqsConsumer/Consumer.cs @@ -0,0 +1,127 @@ +using Amazon.Runtime; +using Amazon.SQS; +using LocalPost.DependencyInjection; +using LocalPost.Flow; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; + +namespace LocalPost.SqsConsumer; + +internal sealed class Consumer(string name, ILogger logger, IAmazonSQS sqs, + ConsumerOptions settings, IHandlerManager> hm) + : IHostedService, IHealthAwareService, IDisposable +{ + private CancellationTokenSource? _execTokenSource; + private Task? _exec; + private Exception? _execException; + private string? _execExceptionDescription; + + private CancellationToken _completionToken = CancellationToken.None; + + private HealthCheckResult Ready => (_execTokenSource, _execution: _exec, _execException) switch + { + (null, _, _) => HealthCheckResult.Unhealthy("Not started"), + (_, { IsCompleted: true }, _) => HealthCheckResult.Unhealthy("Stopped"), + (not null, null, _) => HealthCheckResult.Degraded("Starting"), + (not null, not null, null) => HealthCheckResult.Healthy("Running"), + (_, _, not null) => HealthCheckResult.Unhealthy(_execExceptionDescription, _execException), + }; + + public IHealthCheck ReadinessCheck => HealthChecks.From(() => Ready); + + private async Task RunConsumerAsync( + QueueClient client, Handler> handler, CancellationToken execToken) + { + // (Optionally) wait for app start + + try + { + while (!execToken.IsCancellationRequested) + { + var messages = await client.PullMessages(execToken).ConfigureAwait(false); + await Task.WhenAll(messages + .Select(message => new ConsumeContext(client, message, message.Body)) + .Select(context => handler(context, CancellationToken.None).AsTask())) + .ConfigureAwait(false); + } + } + catch (OperationCanceledException e) when (e.CancellationToken == execToken) + { + // logger.LogInformation("SQS consumer shutdown"); + } + catch (AmazonServiceException e) + { + logger.LogCritical(e, "SQS consumer error: {ErrorCode} (see {HelpLink})", e.ErrorCode, e.HelpLink); + (_execException, _execExceptionDescription) = (e, "SQS consumer failed"); + } + catch (Exception e) + { + logger.LogCritical(e, "SQS message handler error"); + (_execException, _execExceptionDescription) = (e, "Message handler failed"); + } + finally + { + CancelExecution(); // Stop other consumers too + } + } + + public async Task StartAsync(CancellationToken ct) + { + if (_execTokenSource is not null) + throw new InvalidOperationException("Already started"); + + var execTokenSource = _execTokenSource = new CancellationTokenSource(); + + var client = new QueueClient(logger, sqs, settings); + await client.Connect(ct).ConfigureAwait(false); + + var handler = await hm.Start(ct).ConfigureAwait(false); + + _exec = ObserveExecution(); + return; + + async Task ObserveExecution() + { + try + { + var execution = settings.Consumers switch + { + 1 => RunConsumerAsync(client, handler, execTokenSource.Token), + _ => Task.WhenAll(Enumerable + .Range(0, settings.Consumers) + .Select(_ => RunConsumerAsync(client, handler, execTokenSource.Token))) + }; + await execution.ConfigureAwait(false); + + await hm.Stop(_execException, _completionToken).ConfigureAwait(false); + } + finally + { + // Can happen before the service shutdown, in case of an error + logger.LogInformation("SQS consumer stopped"); + } + } + } + + // await _execTokenSource.CancelAsync(); // .NET 8+ + private void CancelExecution() => _execTokenSource?.Cancel(); + + public async Task StopAsync(CancellationToken forceShutdownToken) + { + if (_execTokenSource is null) + throw new InvalidOperationException("Has not been started"); + + logger.LogInformation("Shutting down SQS consumer..."); + + _completionToken = forceShutdownToken; + CancelExecution(); + if (_exec is not null) + await _exec.ConfigureAwait(false); + } + + public void Dispose() + { + _execTokenSource?.Dispose(); + _exec?.Dispose(); + } +} diff --git a/src/LocalPost.SqsConsumer/DependencyInjection/HealthChecksBuilderEx.cs b/src/LocalPost.SqsConsumer/DependencyInjection/HealthChecksBuilderEx.cs new file mode 100644 index 0000000..591cd0e --- /dev/null +++ b/src/LocalPost.SqsConsumer/DependencyInjection/HealthChecksBuilderEx.cs @@ -0,0 +1,22 @@ +using LocalPost.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace LocalPost.SqsConsumer.DependencyInjection; + +[PublicAPI] +public static class HealthChecksBuilderEx +{ + public static IHealthChecksBuilder AddSqsConsumer(this IHealthChecksBuilder builder, + string name, HealthStatus? failureStatus = null, IEnumerable? tags = null) => + builder.Add(HealthChecks.Readiness(name, failureStatus, tags)); + + public static IHealthChecksBuilder AddSqsConsumers(this IHealthChecksBuilder builder, + HealthStatus? failureStatus = null, IEnumerable? tags = null) + { + foreach (var name in builder.Services.GetKeysFor().OfType()) + AddSqsConsumer(builder, name, failureStatus, tags); + + return builder; + } +} diff --git a/src/LocalPost.SqsConsumer/DependencyInjection/ServiceCollectionEx.cs b/src/LocalPost.SqsConsumer/DependencyInjection/ServiceCollectionEx.cs new file mode 100644 index 0000000..8fa53ec --- /dev/null +++ b/src/LocalPost.SqsConsumer/DependencyInjection/ServiceCollectionEx.cs @@ -0,0 +1,14 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace LocalPost.SqsConsumer.DependencyInjection; + +[PublicAPI] +public static class ServiceCollectionEx +{ + public static IServiceCollection AddSqsConsumers(this IServiceCollection services, Action configure) + { + configure(new SqsBuilder(services)); + + return services; + } +} diff --git a/src/LocalPost.SqsConsumer/DependencyInjection/ServiceCollectionExtensions.cs b/src/LocalPost.SqsConsumer/DependencyInjection/ServiceCollectionExtensions.cs deleted file mode 100644 index 71c0961..0000000 --- a/src/LocalPost.SqsConsumer/DependencyInjection/ServiceCollectionExtensions.cs +++ /dev/null @@ -1,83 +0,0 @@ -using Amazon.SQS.Model; -using LocalPost; -using LocalPost.DependencyInjection; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; - -namespace LocalPost.SqsConsumer.DependencyInjection; - -public static class ServiceCollectionExtensions -{ - public static OptionsBuilder AddAmazonSqsMinimalConsumer(this IServiceCollection services, - string name, MessageHandler handler) => - services.AddAmazonSqsConsumer(name, _ => handler); - - public static OptionsBuilder AddAmazonSqsMinimalConsumer(this IServiceCollection services, - string name, Func handler) where TDep1 : notnull => - services.AddAmazonSqsConsumer(name, provider => (context, ct) => - { - var dep1 = provider.GetRequiredService(); - - return handler(dep1, context, ct); - }); - - public static OptionsBuilder AddAmazonSqsMinimalConsumer(this IServiceCollection services, - string name, Func handler) - where TDep1 : notnull - where TDep2 : notnull => - services.AddAmazonSqsConsumer(name, provider => (context, ct) => - { - var dep1 = provider.GetRequiredService(); - var dep2 = provider.GetRequiredService(); - - return handler(dep1, dep2, context, ct); - }); - - public static OptionsBuilder AddAmazonSqsMinimalConsumer(this IServiceCollection services, - string name, Func handler) - where TDep1 : notnull - where TDep2 : notnull - where TDep3 : notnull => - services.AddAmazonSqsConsumer(name, provider => (context, ct) => - { - var dep1 = provider.GetRequiredService(); - var dep2 = provider.GetRequiredService(); - var dep3 = provider.GetRequiredService(); - - return handler(dep1, dep2, dep3, context, ct); - }); - - public static OptionsBuilder AddAmazonSqsConsumer(this IServiceCollection services, - string name) where THandler : IMessageHandler => - services - .AddAmazonSqsConsumer(name, provider => provider.GetRequiredService().Process); - - public static OptionsBuilder AddAmazonSqsConsumer(this IServiceCollection services, - string name, Func> handlerFactory) - { - services.TryAddSingleton, SqsConsumerOptionsResolver>(); - - services.TryAddSingleton(); - services.AddSingleton(provider => ActivatorUtilities.CreateInstance(provider, name)); - - services - .AddCustomBackgroundQueue($"SQS/{name}", - provider => provider.GetSqs(name), - provider => provider.GetSqs(name).Handler(handlerFactory(provider))) - .Configure>( - (options, sqsOptions) => { options.MaxConcurrency = sqsOptions.Get(name).MaxConcurrency; }); - - services.TryAddSingleton(); - services - .AddCustomBackgroundQueue($"SQS/{name}/ProcessedMessages", - provider => provider.GetSqs(name).ProcessedMessages, - provider => provider.GetRequiredService().Process) - .Configure>( - (options, sqsOptions) => { options.MaxConcurrency = sqsOptions.Get(name).MaxConcurrency; }); - - // TODO Health check, metrics - - return services.AddOptions(name).Configure(options => options.QueueName = name); - } -} diff --git a/src/LocalPost.SqsConsumer/DependencyInjection/SqsBuilder.cs b/src/LocalPost.SqsConsumer/DependencyInjection/SqsBuilder.cs new file mode 100644 index 0000000..6668476 --- /dev/null +++ b/src/LocalPost.SqsConsumer/DependencyInjection/SqsBuilder.cs @@ -0,0 +1,50 @@ +using Amazon.SQS; +using LocalPost.DependencyInjection; +using LocalPost.Flow; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace LocalPost.SqsConsumer.DependencyInjection; + +[PublicAPI] +public sealed class SqsBuilder(IServiceCollection services) +{ + public OptionsBuilder Defaults { get; } = services.AddOptions(); + + /// + /// Add an SQS consumer with a custom message handler. + /// + /// Message handler factory. + /// Consumer options builder. + public OptionsBuilder AddConsumer(HandlerManagerFactory> hmf) => + AddConsumer(Options.DefaultName, hmf); + + /// + /// Add an SQS consumer with a custom message handler. + /// + /// Consumer name (should be unique in the application). Also, the default queue name. + /// Message handler factory. + /// Consumer options builder. + public OptionsBuilder AddConsumer(string name, HandlerManagerFactory> hmf) + { + var added = services.TryAddKeyedSingleton(name, (provider, _) => new Consumer(name, + provider.GetLoggerFor(), + provider.GetRequiredService(), + provider.GetOptions(name), + hmf(provider) + )); + + if (!added) + throw new ArgumentException("Consumer is already registered", nameof(name)); + + services.AddHostedService(provider => provider.GetRequiredKeyedService(name)); + + return OptionsFor(name).Configure>((co, defaults) => + { + co.UpdateFrom(defaults.Value); + co.QueueName = name; + }); + } + + public OptionsBuilder OptionsFor(string name) => services.AddOptions(name); +} diff --git a/src/LocalPost.SqsConsumer/HandlerStackEx.cs b/src/LocalPost.SqsConsumer/HandlerStackEx.cs new file mode 100644 index 0000000..2a83086 --- /dev/null +++ b/src/LocalPost.SqsConsumer/HandlerStackEx.cs @@ -0,0 +1,104 @@ +using System.Text.Json; + +namespace LocalPost.SqsConsumer; + +using MessageHmf = HandlerManagerFactory>; +using MessagesHmf = HandlerManagerFactory>>; + +[PublicAPI] +public static class HandlerStackEx +{ + public static HandlerManagerFactory> UseSqsPayload(this HandlerManagerFactory hmf) => + hmf.MapHandler, T>(next => async (context, ct) => + await next(context.Payload, ct).ConfigureAwait(false)); + + public static HandlerManagerFactory>> UseSqsPayload( + this HandlerManagerFactory> hmf) => + hmf.MapHandler>, IReadOnlyCollection>(next => async (batch, ct) => + await next(batch.Select(context => context.Payload).ToArray(), ct).ConfigureAwait(false)); + + public static HandlerManagerFactory> Trace(this HandlerManagerFactory> hmf) => + hmf.TouchHandler(next => async (context, ct) => + { + using var activity = Tracing.StartProcessing(context); + try + { + await next(context, ct).ConfigureAwait(false); + activity?.Success(); + } + catch (Exception e) + { + activity?.Error(e); + throw; + } + }); + + public static HandlerManagerFactory>> Trace( + this HandlerManagerFactory>> hmf) => + hmf.TouchHandler(next => async (batch, ct) => + { + using var activity = Tracing.StartProcessing(batch); + try + { + await next(batch, ct).ConfigureAwait(false); + activity?.Success(); + } + catch (Exception e) + { + activity?.Error(e); + throw; + } + }); + + public static HandlerManagerFactory> Acknowledge( + this HandlerManagerFactory> hmf) => hmf.TouchHandler(next => + async (context, ct) => + { + await next(context, ct).ConfigureAwait(false); + // await context.Client.DeleteMessage(context, ct).ConfigureAwait(false); + await context.DeleteMessage(ct).ConfigureAwait(false); + }); + + public static HandlerManagerFactory>> Acknowledge( + this HandlerManagerFactory>> hmf) => hmf.TouchHandler(next => + async (batch, ct) => + { + await next(batch, ct).ConfigureAwait(false); + if (batch.Count > 0) + await batch.First().Client.DeleteMessages(batch, ct).ConfigureAwait(false); + }); + + public static MessageHmf Deserialize( + this HandlerManagerFactory> hmf, + Func, T>> df) => provider => + { + var handler = hmf(provider); + var deserialize = df(provider); + return handler.Map, ConsumeContext>(next => + async (context, ct) => await next(context.Transform(deserialize), ct).ConfigureAwait(false)); + }; + + public static MessagesHmf Deserialize( + this HandlerManagerFactory>> hmf, + Func, T>> df) => provider => + { + var handler = hmf(provider); + var deserialize = df(provider); + return handler.Map>, IReadOnlyCollection>>(next => + async (batch, ct) => + { + var modBatch = batch.Select(context => context.Transform(deserialize)).ToArray(); + await next(modBatch, ct).ConfigureAwait(false); + }); + }; + + public static MessageHmf DeserializeJson( + this HandlerManagerFactory> hmf, + JsonSerializerOptions? options = null) => + hmf.Deserialize(_ => context => JsonSerializer.Deserialize(context.Payload, options)!); + + public static MessagesHmf DeserializeJson( + this HandlerManagerFactory>> hmf, + JsonSerializerOptions? options = null) => + hmf.Deserialize(_ => context => JsonSerializer.Deserialize(context.Payload, options)!); +} diff --git a/src/LocalPost.SqsConsumer/ISqsMessageHandler.cs b/src/LocalPost.SqsConsumer/ISqsMessageHandler.cs deleted file mode 100644 index 0b454a0..0000000 --- a/src/LocalPost.SqsConsumer/ISqsMessageHandler.cs +++ /dev/null @@ -1,8 +0,0 @@ -using Amazon.SQS.Model; -using LocalPost; - -namespace LocalPost.SqsConsumer; - -public interface ISqsMessageHandler : IMessageHandler -{ -} diff --git a/src/LocalPost.SqsConsumer/LocalPost.SqsConsumer.csproj b/src/LocalPost.SqsConsumer/LocalPost.SqsConsumer.csproj index 7feb6cf..c038360 100644 --- a/src/LocalPost.SqsConsumer/LocalPost.SqsConsumer.csproj +++ b/src/LocalPost.SqsConsumer/LocalPost.SqsConsumer.csproj @@ -1,51 +1,38 @@ - netstandard2.0 - true - - false + net6;net8 LocalPost.SqsConsumer - background;task;queue;amazon;sns;aws + Local (in-process) background queue for sending to Amazon SNS. - Alexey Shokov + background;task;queue;amazon;sqs;aws README.md - MIT - https://github.com/alexeyshockov/LocalPost - git - true - + - - - true - - - - true - true - true - true - snupkg + + + + - - true - + + + - - - - - - + + + + diff --git a/src/LocalPost.SqsConsumer/MetricsReporter.cs b/src/LocalPost.SqsConsumer/MetricsReporter.cs new file mode 100644 index 0000000..862e5b9 --- /dev/null +++ b/src/LocalPost.SqsConsumer/MetricsReporter.cs @@ -0,0 +1,9 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Reflection; + +namespace LocalPost.SqsConsumer; + +// TODO Metrics +// .NET docs on metric instrumentation: https://learn.microsoft.com/en-us/dotnet/core/diagnostics/metrics-instrumentation +// OpenTelemetry semantic conventions: https://opentelemetry.io/docs/specs/semconv/messaging/messaging-metrics/#consumer-metrics diff --git a/src/LocalPost.SqsConsumer/Options.cs b/src/LocalPost.SqsConsumer/Options.cs new file mode 100644 index 0000000..c106416 --- /dev/null +++ b/src/LocalPost.SqsConsumer/Options.cs @@ -0,0 +1,138 @@ +using System.ComponentModel.DataAnnotations; +using JetBrains.Annotations; + +namespace LocalPost.SqsConsumer; + +/// +/// General SQS settings. +/// +[PublicAPI] +public sealed class EndpointOptions +{ + /// + /// Time to wait for available messages in the queue. 0 is short pooling, where 1..20 activates long pooling. + /// Default is 20. + /// + /// + /// Amazon SQS short and long polling + /// + /// + /// Setting up long polling + /// + [Range(0, 20)] + public byte WaitTimeSeconds { get; set; } = 20; + + /// + /// The maximum number of messages to return. Amazon SQS never returns more messages than this value (however, + /// fewer messages might be returned). Valid values: 1 to 10. Default is 1. + /// + /// + /// Amazon SQS short and long polling + /// + /// + /// Setting up long polling + /// + [Range(1, 10)] + public byte MaxNumberOfMessages { get; set; } = 10; + + // User specific thing... +// /// +// /// Message processing timeout. If not set, IAmazonSQS.GetQueueAttributesAsync() will be used once, +// /// to get VisibilityTimeout for the queue. If it is not available, default value of 10 seconds will be used. +// /// +// /// +// /// Amazon SQS visibility timeout +// /// +// [Range(1, int.MaxValue)] +// public int? TimeoutMilliseconds { get; set; } + + public List AttributeNames { get; set; } = ["All"]; + + public List MessageAttributeNames { get; set; } = ["All"]; +} + +/// +/// SQS queue consumer settings. +/// +[PublicAPI] +public sealed class ConsumerOptions +{ + [Range(1, ushort.MaxValue)] + // public ushort MaxConcurrency { get; set; } = 10; + public ushort Consumers { get; set; } = 1; + + /// + /// Time to wait for available messages in the queue. 0 is short pooling, where 1..20 activates long pooling. + /// Default is 20. + /// + /// + /// Amazon SQS short and long polling + /// + /// + /// Setting up long polling + /// + [Range(0, 20)] + public byte WaitTimeSeconds { get; set; } = 20; + + /// + /// The maximum number of messages to return. Valid values: 1 to 10. Default is 10. + /// + /// Amazon SQS never returns more messages than this value (however, fewer messages might be returned). + /// + /// All the returned messages will be processed concurrently. + /// + /// + /// Amazon SQS short and long polling + /// + /// + /// Setting up long polling + /// + [Range(1, 10)] + public byte MaxNumberOfMessages { get; set; } = 10; + + public List AttributeNames { get; set; } = ["All"]; + + public List MessageAttributeNames { get; set; } = ["All"]; + + internal void UpdateFrom(EndpointOptions global) + { + // MaxConcurrency = other.MaxConcurrency; + // Prefetch = other.Prefetch; + WaitTimeSeconds = global.WaitTimeSeconds; + MaxNumberOfMessages = global.MaxNumberOfMessages; + AttributeNames = global.AttributeNames; + MessageAttributeNames = global.MessageAttributeNames; + } + + internal void UpdateFrom(ConsumerOptions other) + { + WaitTimeSeconds = other.WaitTimeSeconds; + MaxNumberOfMessages = other.MaxNumberOfMessages; + AttributeNames = other.AttributeNames; + MessageAttributeNames = other.MessageAttributeNames; + QueueName = other.QueueName; + _queueUrl = other._queueUrl; + } + + [Required] + public string QueueName { get; set; } = null!; + + private string? _queueUrl; + + /// + /// If not set, IAmazonSQS.GetQueueUrlAsync(QueueName) will be used once, to get the actual URL of the queue. + /// + [Url] + public string? QueueUrl + { + get => _queueUrl; + set + { + _queueUrl = value; + + // Extract name (MyQueue) from an URL (https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue) + if (Uri.TryCreate(value, UriKind.Absolute, out var url) && url.Segments.Length >= 3) + QueueName = url.Segments[2]; + } + } +} diff --git a/src/LocalPost.SqsConsumer/ProcessedMessages.cs b/src/LocalPost.SqsConsumer/ProcessedMessages.cs deleted file mode 100644 index a5a4117..0000000 --- a/src/LocalPost.SqsConsumer/ProcessedMessages.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System.Threading.Channels; -using Amazon.SQS; -using Amazon.SQS.Model; -using LocalPost; - -namespace LocalPost.SqsConsumer; - -internal sealed class ProcessedMessages : IBackgroundQueue, IAsyncEnumerable -{ - private readonly string _queueUrl; - private readonly Channel _messages = - Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false, - }); - - public ProcessedMessages(string queueUrl) - { - _queueUrl = queueUrl; - } - - public ValueTask Enqueue(Message item, CancellationToken ct = default) => _messages.Writer.WriteAsync( - new DeleteMessageBatchRequestEntry(Guid.NewGuid().ToString(), item.ReceiptHandle), ct); - - public IAsyncEnumerator GetAsyncEnumerator(CancellationToken ct = default) => - _messages.Reader.ReadAllAsync(ct).Batch(() => new SqsDeleteBatchBuilder(_queueUrl)).GetAsyncEnumerator(ct); -} - -internal sealed class ProcessedMessagesHandler : IMessageHandler -{ - private readonly IAmazonSQS _sqs; - - public ProcessedMessagesHandler(IAmazonSQS sqs) - { - _sqs = sqs; - } - - public async Task Process(DeleteMessageBatchRequest payload, CancellationToken ct) - { - var response = await _sqs.DeleteMessageBatchAsync(payload, ct); - if (response.Failed.Any()) - { - // TODO Log failures - } - } -} diff --git a/src/LocalPost.SqsConsumer/QueueClient.cs b/src/LocalPost.SqsConsumer/QueueClient.cs new file mode 100644 index 0000000..220b3ce --- /dev/null +++ b/src/LocalPost.SqsConsumer/QueueClient.cs @@ -0,0 +1,117 @@ +using Amazon.Runtime; +using Amazon.SQS; +using Amazon.SQS.Model; +using Polly; +using Polly.Retry; + +namespace LocalPost.SqsConsumer; + +internal sealed class QueueClient(ILogger logger, IAmazonSQS sqs, ConsumerOptions options) +{ + public string QueueName => options.QueueName; + + private GetQueueAttributesResponse? _queueAttributes; + + private readonly ResiliencePipeline _pipeline = new ResiliencePipelineBuilder() + .AddRetry(new RetryStrategyOptions + { + ShouldHandle = new PredicateBuilder() + .Handle(e => e.Retryable is not null), + + Delay = TimeSpan.FromSeconds(1), + MaxRetryAttempts = byte.MaxValue, + BackoffType = DelayBackoffType.Exponential, + + // Initially, aim for an exponential backoff, but after a certain number of retries, set a maximum delay + MaxDelay = TimeSpan.FromMinutes(1), + UseJitter = true + }) + .Build(); + + // TODO Use + public TimeSpan? MessageVisibilityTimeout => _queueAttributes?.VisibilityTimeout switch + { + > 0 => TimeSpan.FromSeconds(_queueAttributes.VisibilityTimeout), + _ => null + }; + + private string? _queueUrl; + private string QueueUrl => _queueUrl ?? throw new InvalidOperationException("SQS queue client is not connected"); + + public async Task Connect(CancellationToken ct) + { + if (string.IsNullOrEmpty(options.QueueUrl)) + // Checking for a possible error in the response would be also good... + _queueUrl = (await sqs.GetQueueUrlAsync(options.QueueName, ct).ConfigureAwait(false)).QueueUrl; + + await FetchQueueAttributes(ct).ConfigureAwait(false); + } + + private async Task FetchQueueAttributes(CancellationToken ct) + { + try + { + // Checking for a possible error in the response would be also good... + _queueAttributes = await sqs.GetQueueAttributesAsync(QueueUrl, ["All"], ct).ConfigureAwait(false); + } + catch (OperationCanceledException e) when (e.CancellationToken == ct) + { + throw; + } + catch (Exception e) + { + logger.LogWarning(e, "Cannot fetch attributes for SQS {Queue}", options.QueueName); + } + } + + public async Task> PullMessages(CancellationToken ct) => + await _pipeline.ExecuteAsync(PullMessagesCore, ct).ConfigureAwait(false); + + private async ValueTask> PullMessagesCore(CancellationToken ct) + { + using var activity = Tracing.StartReceiving(this); + + try + { + // AWS SDK handles network failures, see + // https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html + var response = await sqs.ReceiveMessageAsync(new ReceiveMessageRequest + { + QueueUrl = QueueUrl, + WaitTimeSeconds = options.WaitTimeSeconds, + MaxNumberOfMessages = options.MaxNumberOfMessages, + AttributeNames = options.AttributeNames, + MessageAttributeNames = options.MessageAttributeNames, + }, ct).ConfigureAwait(false); + + activity?.SetTagsFor(response); + activity?.Success(); + + return response.Messages; + } + catch (OperationCanceledException e) when (e.CancellationToken == ct) + { + throw; + } + catch (Exception e) + { + activity?.Error(e); + throw; + } + } + + public async Task DeleteMessage(ConsumeContext context, CancellationToken ct = default) + { + using var activity = Tracing.StartSettling(context); + await sqs.DeleteMessageAsync(QueueUrl, context.ReceiptHandle, ct).ConfigureAwait(false); + + // TODO Log failures?.. + } + + public async Task DeleteMessages(IEnumerable> batch, CancellationToken ct = default) + { + // TODO DeleteMessageBatch + foreach (var context in batch) + await sqs.DeleteMessageAsync(QueueUrl, context.ReceiptHandle, ct).ConfigureAwait(false); + } +} diff --git a/src/LocalPost.SqsConsumer/README.md b/src/LocalPost.SqsConsumer/README.md new file mode 100644 index 0000000..7b1b69c --- /dev/null +++ b/src/LocalPost.SqsConsumer/README.md @@ -0,0 +1,16 @@ +# LocalPost Amazon SQS Consumer + +## IAM Permissions + +Only `sqs:ReceiveMessage` is required to run a queue consumer. To use additional features also require: +- `sqs:GetQueueUrl` (to use queue names instead of the full URLs) +- `sqs:GetQueueAttributes` (to fetch queue attributes on startup) + +## Batching + +The first thing to note when using batch processing with SQS: make sure that the queue +[visibility timeout](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html) +is greater than the batch window. Otherwise, the messages will be re-queued before the batch is processed. + +Most of other [AWS Lambda recommendations](https://docs.aws.amazon.com/lambda/latest/dg/services-sqs-configure.html) +also apply. diff --git a/src/LocalPost.SqsConsumer/SqsConsumerOptions.cs b/src/LocalPost.SqsConsumer/SqsConsumerOptions.cs deleted file mode 100644 index 352b734..0000000 --- a/src/LocalPost.SqsConsumer/SqsConsumerOptions.cs +++ /dev/null @@ -1,106 +0,0 @@ -using System.Collections.Immutable; -using System.ComponentModel.DataAnnotations; -using Amazon.SQS; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace LocalPost.SqsConsumer; - -/// -/// General SQS consumer settings -/// -public sealed record SqsConsumerOptions -{ - public static readonly ImmutableArray AllAttributes = ImmutableArray.Create("All"); - public static readonly ImmutableArray AllMessageAttributes = ImmutableArray.Create("All"); - - public const ushort DefaultTimeout = 30; - - /// - /// How many messages to process in parallel. - /// - [Required] public ushort MaxConcurrency { get; set; } = 10; - - [Required] public string QueueName { get; set; } = null!; - - /// - /// If not set, IAmazonSQS.GetQueueUrlAsync(QueueName) will be used once, to get the actual URL of the queue. - /// - [Url] public string? QueueUrl { get; set; } - - /// - /// Time to wait for available messages in the queue. - /// - /// - /// Amazon SQS short and long polling - /// - [Range(0, 60)] public byte WaitTimeSeconds { get; set; } = 20; - - [Range(1, 10)] public byte MaxNumberOfMessages { get; set; } = 10; - - /// - /// Message processing timeout, in seconds. If not set, IAmazonSQS.GetQueueAttributesAsync() will be used once, to get - /// VisibilityTimeout for the queue. - /// - /// - /// Amazon SQS visibility timeout - /// - [Range(1, 43200)] - public ushort Timeout { get; set; } -} - -internal sealed class SqsConsumerOptionsResolver : IPostConfigureOptions -{ - private readonly ILogger _logger; - private readonly IAmazonSQS _sqs; - - public SqsConsumerOptionsResolver(ILogger logger, IAmazonSQS sqs) - { - _logger = logger; - _sqs = sqs; - } - - public void PostConfigure(string name, SqsConsumerOptions options) - { - if (string.IsNullOrEmpty(options.QueueUrl)) - FetchQueueUrl(options).Wait(); - - if (options.Timeout == 0) // Try to get it from the SQS settings if not set - FetchVisibilityTimeout(options).Wait(); - } - - private async Task FetchQueueUrl(SqsConsumerOptions options) - { - try - { - // Checking possible errors in the response would be good - options.QueueUrl = (await _sqs.GetQueueUrlAsync(options.QueueName)).QueueUrl; - } - catch (Exception e) - { - // TODO Wrap in our own exception - throw new ArgumentException($"Cannot fetch SQS URL for {options.QueueName}", nameof(options), e); - } - } - - private async Task FetchVisibilityTimeout(SqsConsumerOptions options) - { - try - { - // Checking possible errors in the response would be good - var queueAttributes = await _sqs - .GetQueueAttributesAsync(options.QueueUrl, SqsConsumerOptions.AllAttributes.ToList()); - - options.Timeout = queueAttributes.VisibilityTimeout switch - { - > 0 => (ushort) queueAttributes.VisibilityTimeout, - _ => SqsConsumerOptions.DefaultTimeout - }; - } - catch (Exception e) - { - _logger.LogWarning(e, "Cannot fetch SQS attributes for {Queue}", options.QueueName); - options.Timeout = SqsConsumerOptions.DefaultTimeout; - } - } -} diff --git a/src/LocalPost.SqsConsumer/SqsDeleteBatchBuilder.cs b/src/LocalPost.SqsConsumer/SqsDeleteBatchBuilder.cs deleted file mode 100644 index d33294d..0000000 --- a/src/LocalPost.SqsConsumer/SqsDeleteBatchBuilder.cs +++ /dev/null @@ -1,30 +0,0 @@ -using Amazon.SQS.Model; - -namespace LocalPost.SqsConsumer; - -internal sealed class SqsDeleteBatchBuilder : BatchBuilder -{ - private readonly DeleteMessageBatchRequest _batchRequest; - - public SqsDeleteBatchBuilder(string queueUrl) : base(TimeSpan.FromSeconds(1)) // TODO Configurable - { - _batchRequest = new DeleteMessageBatchRequest { QueueUrl = queueUrl }; - } - - public override bool IsEmpty => _batchRequest.Entries.Count == 0; - - private bool CanFit(DeleteMessageBatchRequestEntry entry) => _batchRequest.Entries.Count <= 10; - - public override bool TryAdd(DeleteMessageBatchRequestEntry entry) - { - var canFit = CanFit(entry); - if (!canFit) - return false; - - _batchRequest.Entries.Add(entry); - - return true; - } - - public override DeleteMessageBatchRequest Build() => _batchRequest; -} diff --git a/src/LocalPost.SqsConsumer/SqsPuller.cs b/src/LocalPost.SqsConsumer/SqsPuller.cs deleted file mode 100644 index a8c10e9..0000000 --- a/src/LocalPost.SqsConsumer/SqsPuller.cs +++ /dev/null @@ -1,83 +0,0 @@ -using System.Collections.Immutable; -using Amazon.SQS; -using Amazon.SQS.Model; -using LocalPost; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; - -namespace LocalPost.SqsConsumer; - -internal static partial class ServiceProviderExtensions -{ - public static SqsPuller GetSqs(this IServiceProvider provider, string name) => - provider.GetRequiredService()[name]; -} - -internal sealed class SqsAccessor -{ - private readonly IReadOnlyDictionary _registry; - - public SqsAccessor(IEnumerable registry) - { - _registry = registry.ToImmutableDictionary(x => x.Name, x => x); - } - - public SqsPuller this[string name] => _registry[name]; -} - -internal sealed class SqsPuller : IAsyncEnumerable -{ - private readonly IAmazonSQS _sqs; - private readonly SqsConsumerOptions _options; - - private readonly IBackgroundQueue _processedMessages; - - public SqsPuller(IAmazonSQS sqs, string name, IOptionsMonitor options) - { - _sqs = sqs; - _options = options.Get(name); - - var processedMessages = new ProcessedMessages(_options.QueueUrl); - _processedMessages = processedMessages; - ProcessedMessages = processedMessages; - - Name = name; - } - - public string Name { get; } - - public IAsyncEnumerable ProcessedMessages { get; } - - public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken ct = default) - { - var attributeNames = SqsConsumerOptions.AllAttributes.ToList(); // TODO Configurable - var messageAttributeNames = SqsConsumerOptions.AllMessageAttributes.ToList(); // TODO Configurable - - while (!ct.IsCancellationRequested) - { - // AWS SDK handles network failures, see - // https://docs.aws.amazon.com/sdkref/latest/guide/feature-retry-behavior.html - var receiveMessageResponse = await _sqs.ReceiveMessageAsync(new ReceiveMessageRequest - { - QueueUrl = _options.QueueUrl, - WaitTimeSeconds = _options.WaitTimeSeconds, - MaxNumberOfMessages = _options.MaxNumberOfMessages, - AttributeNames = attributeNames, - MessageAttributeNames = messageAttributeNames, - }, ct); - - foreach (var message in receiveMessageResponse.Messages) - yield return message; - } - } - - public MessageHandler Handler(MessageHandler handler) => async (payload, ct) => - { - await handler(payload, ct); - - // Won't be deleted in case of an exception in the handler - await _processedMessages.Enqueue(payload, ct); - - // Extend message's VisibilityTimeout in case of long processing?.. - }; -} diff --git a/src/LocalPost.SqsConsumer/Tracing.cs b/src/LocalPost.SqsConsumer/Tracing.cs new file mode 100644 index 0000000..1a68bfb --- /dev/null +++ b/src/LocalPost.SqsConsumer/Tracing.cs @@ -0,0 +1,142 @@ +using System.Diagnostics; +using System.Reflection; +using Amazon.SQS.Model; + +namespace LocalPost.SqsConsumer; + +internal static class MessageUtils +{ + public static void ExtractTraceField(object? carrier, string fieldName, + out string? fieldValue, out IEnumerable? fieldValues) + { + fieldValues = null; + fieldValue = null; + if (carrier is not Message message) + return; + + fieldValue = message.MessageAttributes.TryGetValue(fieldName, out var attribute) + ? attribute.StringValue + : null; + } +} + +internal static class SqsActivityExtensions +{ + public static void AcceptDistributedTracingFrom(this Activity activity, Message message) + { + var propagator = DistributedContextPropagator.Current; + propagator.ExtractTraceIdAndState(message, MessageUtils.ExtractTraceField, + out var traceParent, out var traceState); + + if (string.IsNullOrEmpty(traceParent)) + return; + activity.SetParentId(traceParent!); + if (!string.IsNullOrEmpty(traceState)) + activity.TraceStateString = traceState; + + var baggage = propagator.ExtractBaggage(message, MessageUtils.ExtractTraceField); + if (baggage is null) + return; + foreach (var baggageItem in baggage) + activity.AddBaggage(baggageItem.Key, baggageItem.Value); + } + + public static void SetDefaultTags(this Activity? activity, QueueClient client) + { + // See https://opentelemetry.io/docs/specs/semconv/messaging/messaging-spans/#messaging-attributes + activity?.SetTag("messaging.system", "aws_sqs"); + + activity?.SetTag("messaging.destination.name", client.QueueName); + + // activity?.SetTag("messaging.client_id", "service_name"); + // activity?.SetTag("server.address", client.QueueUrl.Host); + // activity?.SetTag("server.port", client.QueueUrl.Port); + } + + public static Activity? SetTagsFor(this Activity? activity, ConsumeContext context) => + activity?.SetTag("messaging.message.id", context.MessageId); + + public static Activity? SetTagsFor(this Activity? activity, IReadOnlyCollection> batch) => + activity?.SetTag("messaging.batch.message_count", batch.Count); + + public static Activity? SetTagsFor(this Activity? activity, ReceiveMessageResponse response) => + activity?.SetTag("messaging.batch.message_count", response.Messages.Count); +} + +// Based on Semantic Conventions 1.30.0, see +// https://opentelemetry.io/docs/specs/semconv/messaging/messaging-spans/ +// Also Npgsql as an inspiration: +// - https://github.com/npgsql/npgsql/blob/main/src/Npgsql/NpgsqlActivitySource.cs +// - https://github.com/npgsql/npgsql/blob/main/src/Npgsql/NpgsqlCommand.cs +internal static class Tracing +{ + private static readonly ActivitySource Source; + + public static bool IsEnabled => Source.HasListeners(); + + static Tracing() + { + var assembly = Assembly.GetExecutingAssembly(); + var version = assembly.GetName().Version?.ToString() ?? "0.0.0"; + Source = new ActivitySource("LocalPost.SqsConsumer", version); + } + + public static Activity? StartProcessing(IReadOnlyCollection> batch) + { + Debug.Assert(batch.Count > 0); + var client = batch.First().Client; + var activity = Source.StartActivity($"process {client.QueueName}", ActivityKind.Consumer); + if (activity is { IsAllDataRequested: true }) + { + activity?.SetTag("messaging.operation.type", "process"); + activity.SetDefaultTags(client); + activity.SetTagsFor(batch); + // TODO Distributed tracing (OTEL links) + } + + activity?.Start(); + + return activity; + } + + public static Activity? StartProcessing(ConsumeContext context) + { + var activity = Source.StartActivity($"process {context.Client.QueueName}", ActivityKind.Consumer); + if (activity is { IsAllDataRequested: true }) + { + activity.SetTag("messaging.operation.type", "process"); + activity.SetDefaultTags(context.Client); + activity.SetTagsFor(context); + activity.AcceptDistributedTracingFrom(context.Message); + } + + activity?.Start(); + + return activity; + } + + public static Activity? StartSettling(ConsumeContext context) + { + var activity = Source.StartActivity($"settle {context.Client.QueueName}", ActivityKind.Consumer); + if (activity is not { IsAllDataRequested: true }) + return activity; + + activity.SetTag("messaging.operation.type", "settle"); + activity.SetDefaultTags(context.Client); + activity.SetTag("messaging.message.id", context.MessageId); + + return activity; + } + + public static Activity? StartReceiving(QueueClient client) + { + var activity = Source.StartActivity($"receive {client.QueueName}", ActivityKind.Consumer); + if (activity is not { IsAllDataRequested: true }) + return activity; + + activity.SetTag("messaging.operation.type", "receive"); + activity.SetDefaultTags(client); + + return activity; + } +} diff --git a/src/LocalPost.SqsConsumer/globalusings.cs b/src/LocalPost.SqsConsumer/globalusings.cs new file mode 100644 index 0000000..2f865c2 --- /dev/null +++ b/src/LocalPost.SqsConsumer/globalusings.cs @@ -0,0 +1,3 @@ +global using JetBrains.Annotations; +global using System.Diagnostics.CodeAnalysis; +global using Microsoft.Extensions.Logging; diff --git a/src/LocalPost/ActivityEx.cs b/src/LocalPost/ActivityEx.cs new file mode 100644 index 0000000..a59bba2 --- /dev/null +++ b/src/LocalPost/ActivityEx.cs @@ -0,0 +1,27 @@ +using System.Diagnostics; + +namespace LocalPost; + +internal static class ActivityEx +{ + // See https://github.com/open-telemetry/opentelemetry-dotnet/blob/core-1.8.1/src/OpenTelemetry.Api/Trace/ActivityExtensions.cs#L81-L105 + public static Activity? Error(this Activity? activity, Exception ex, bool escaped = true) + { + activity?.SetTag("otel.status_code", "ERROR"); + // activity.SetTag("otel.status_description", ex is PostgresException pgEx ? pgEx.SqlState : ex.Message); + activity?.SetTag("otel.status_description", ex.Message); + + var tags = new ActivityTagsCollection + { + { "exception.type", ex.GetType().FullName }, + { "exception.message", ex.Message }, + { "exception.stacktrace", ex.ToString() }, + { "exception.escaped", escaped } + }; + activity?.AddEvent(new ActivityEvent("exception", tags: tags)); + + return activity; + } + + public static Activity? Success(this Activity? activity) => activity?.SetTag("otel.status_code", "OK"); +} diff --git a/src/LocalPost/AppHealthSupervisor.cs b/src/LocalPost/AppHealthSupervisor.cs new file mode 100644 index 0000000..a01a47f --- /dev/null +++ b/src/LocalPost/AppHealthSupervisor.cs @@ -0,0 +1,35 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; + +namespace LocalPost; + +[UsedImplicitly] +internal sealed class AppHealthSupervisor(ILogger logger, + HealthCheckService healthChecker, IHostApplicationLifetime appLifetime) : BackgroundService +{ + public TimeSpan CheckInterval { get; init; } = TimeSpan.FromSeconds(1); + public int ExitCode { get; init; } = 1; + public IImmutableSet Tags { get; init; } = ImmutableHashSet.Empty; + + private Task Check(CancellationToken ct = default) => Tags.Count == 0 + ? healthChecker.CheckHealthAsync(ct) + : healthChecker.CheckHealthAsync(hcr => Tags.IsSubsetOf(hcr.Tags), ct); + + protected override async Task ExecuteAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + var result = await Check(ct).ConfigureAwait(false); + if (result.Status == HealthStatus.Unhealthy) + { + logger.LogError("Health check failed, stopping the application..."); + appLifetime.StopApplication(); + Environment.ExitCode = ExitCode; + break; + } + + await Task.Delay(CheckInterval, ct).ConfigureAwait(false); + } + } +} diff --git a/src/LocalPost/AsyncEnumerable.cs b/src/LocalPost/AsyncEnumerable.cs deleted file mode 100644 index 2784b1b..0000000 --- a/src/LocalPost/AsyncEnumerable.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace LocalPost; - -internal static class AsyncEnumerableEx -{ - public static IAsyncEnumerable Batch(this IAsyncEnumerable source, - BatchBuilderFactory factory) => new BatchingAsyncEnumerable(source, factory); - - public static IAsyncEnumerable Merge(this IEnumerable> sources) => - new AsyncEnumerableMerger(sources); -} diff --git a/src/LocalPost/AsyncEnumerableMerger.cs b/src/LocalPost/AsyncEnumerableMerger.cs deleted file mode 100644 index df92a82..0000000 --- a/src/LocalPost/AsyncEnumerableMerger.cs +++ /dev/null @@ -1,87 +0,0 @@ -using System.Collections.Immutable; -using System.Diagnostics.CodeAnalysis; -using System.Threading.Channels; - -namespace LocalPost; - -internal sealed class AsyncEnumerableMerger : IAsyncEnumerable, IDisposable -{ - private readonly ConcurrentSet> _sources; - - public AsyncEnumerableMerger(bool permanent = false) : this(ImmutableArray>.Empty, permanent) - { - } - - public AsyncEnumerableMerger(IEnumerable> sources, bool permanent = false) - { - if (permanent) - // This one IAsyncEnumerable will be there forever, so the wait will be indefinite (even if all other - // sources are completed) - sources = sources.Prepend(Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = false, - SingleWriter = true - }).Reader.ReadAllAsync()); - - _sources = new ConcurrentSet>(sources); - } - - public void Add(IAsyncEnumerable source) => _sources.Add(source); - - [SuppressMessage("ReSharper", "PossibleMultipleEnumeration")] - public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken ct = default) - { - async Task<(IAsyncEnumerable, IAsyncEnumerator, bool)> Wait(IAsyncEnumerable source, IAsyncEnumerator enumerator) => - (source, enumerator, !await enumerator.MoveNextAsync()); - - var sourcesSnapshot = _sources.Elements; - var waits = sourcesSnapshot - .Select(source => Wait(source, source.GetAsyncEnumerator(ct))) - .ToImmutableArray(); - - while (waits.Length > 0) - { - var modificationTrigger = _sources.Modification; - var waitTrigger = Task.WhenAny(waits); - - await Task.WhenAny(waitTrigger, modificationTrigger); - - if (waitTrigger.IsCompleted) // New element - { - var completedWait = await waitTrigger; - var (source, enumerator, completed) = await completedWait; - - if (!completed) - { - yield return enumerator.Current; - waits = waits.Replace(completedWait, Wait(source, enumerator)); - } - else - { - waits = waits.Remove(completedWait); - sourcesSnapshot = _sources.Remove(source); - } - } - - // Always check modification trigger explicitly, as both task can complete when the control is back - // (in this case we can miss the trigger completely) - // ReSharper disable once InvertIf - if (modificationTrigger.IsCompleted) - // Wait() _somehow_ can give the control away, so loop until we sure that all the changes are handled - while (sourcesSnapshot != _sources.Elements) - { - var previousSourcesSnapshot = sourcesSnapshot; - sourcesSnapshot = _sources.Elements; - - // ReSharper disable once ForeachCanBeConvertedToQueryUsingAnotherGetEnumerator - foreach (var newSource in sourcesSnapshot.Except(previousSourcesSnapshot)) - waits = waits.Add(Wait(newSource, newSource.GetAsyncEnumerator(ct))); - } - } - } - - public void Dispose() - { - _sources.Dispose(); - } -} diff --git a/src/LocalPost/BackgroundJobQueue.cs b/src/LocalPost/BackgroundJobQueue.cs deleted file mode 100644 index 0745202..0000000 --- a/src/LocalPost/BackgroundJobQueue.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Threading.Channels; - -namespace LocalPost; - -public delegate Task Job(CancellationToken ct); - -public interface IBackgroundJobQueue : IBackgroundQueue -{ -} - -internal sealed class BackgroundJobQueue : IBackgroundJobQueue, IAsyncEnumerable -{ - private readonly Channel _messages = Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false, - }); - - public ValueTask Enqueue(Job item, CancellationToken ct = default) => _messages.Writer.WriteAsync(item, ct); - - public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) => - _messages.Reader.ReadAllAsync(cancellationToken).GetAsyncEnumerator(cancellationToken); -} diff --git a/src/LocalPost/BackgroundQueue.cs b/src/LocalPost/BackgroundQueue.cs index 2100f38..4900a82 100644 --- a/src/LocalPost/BackgroundQueue.cs +++ b/src/LocalPost/BackgroundQueue.cs @@ -1,37 +1,22 @@ using System.Threading.Channels; +using LocalPost.BackgroundQueue; namespace LocalPost; -public interface IBackgroundQueue +/// +/// Background queue publisher. +/// +/// Payload type. +public interface IBackgroundQueue { - ValueTask Enqueue(T item, CancellationToken ct = default); -} - -public interface IBackgroundQueueReader -{ - public ChannelReader Reader { get; } -} + ValueTask Enqueue(T payload, CancellationToken ct = default); -public interface IMessageHandler -{ - Task Process(TOut payload, CancellationToken ct); + ChannelWriter> Writer { get; } } -public delegate Task MessageHandler(T context, CancellationToken ct); - +public delegate Task BackgroundJob(CancellationToken ct); - -// Simplest background queue -public sealed class BackgroundQueue : IBackgroundQueue, IAsyncEnumerable -{ - private readonly Channel _messages = Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = false, - SingleWriter = false, - }); - - public ValueTask Enqueue(T item, CancellationToken ct = default) => _messages.Writer.WriteAsync(item, ct); - - public IAsyncEnumerator GetAsyncEnumerator(CancellationToken ct = default) => - _messages.Reader.ReadAllAsync(ct).GetAsyncEnumerator(ct); -} +/// +/// Just a convenient alias for queue. +/// +public interface IBackgroundJobQueue : IBackgroundQueue; diff --git a/src/LocalPost/BackgroundQueue/BackgroundJobQueue.cs b/src/LocalPost/BackgroundQueue/BackgroundJobQueue.cs new file mode 100644 index 0000000..a9520f9 --- /dev/null +++ b/src/LocalPost/BackgroundQueue/BackgroundJobQueue.cs @@ -0,0 +1,12 @@ +using System.Threading.Channels; + +namespace LocalPost.BackgroundQueue; + +// Just a proxy to the actual queue, needed to expose IBackgroundJobQueue +[UsedImplicitly] +internal sealed class BackgroundJobQueue(IBackgroundQueue queue) : IBackgroundJobQueue +{ + public ValueTask Enqueue(BackgroundJob payload, CancellationToken ct = default) => queue.Enqueue(payload, ct); + + public ChannelWriter> Writer => queue.Writer; +} diff --git a/src/LocalPost/BackgroundQueue/BackgroundQueue.cs b/src/LocalPost/BackgroundQueue/BackgroundQueue.cs new file mode 100644 index 0000000..0eafbed --- /dev/null +++ b/src/LocalPost/BackgroundQueue/BackgroundQueue.cs @@ -0,0 +1,65 @@ +using System.Threading.Channels; +using LocalPost.DependencyInjection; +using LocalPost.Flow; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; + +namespace LocalPost.BackgroundQueue; + +internal sealed class BackgroundQueue(ILogger> logger, QueueOptions settings, + Channel> channel, IHandlerManager> hm) + : IBackgroundQueue, IHostedService, IHealthAwareService, IDisposable +{ + public IHealthCheck ReadinessCheck => HealthChecks.From(() => _runner switch + { + not null => _runner.Ready, + _ => HealthCheckResult.Unhealthy("Background queue has not started yet"), + }); + + public ValueTask Enqueue(T payload, CancellationToken ct = default) => channel.Writer.WriteAsync(payload, ct); + + public ChannelWriter> Writer => channel.Writer; + + private ChannelRunner, ConsumeContext>? _runner; + + private ChannelRunner, ConsumeContext> CreateRunner(Handler> handler) + { + return new ChannelRunner, ConsumeContext>(channel, Consume, hm) + { Consumers = settings.MaxConcurrency }; + + async Task Consume(CancellationToken execToken) + { + await foreach (var message in channel.Reader.ReadAllAsync(execToken).ConfigureAwait(false)) + await handler(message, CancellationToken.None).ConfigureAwait(false); + } + } + + public async Task StartAsync(CancellationToken ct) + { + var handler = await hm.Start(ct).ConfigureAwait(false); + _runner = CreateRunner(handler); + await _runner.Start(ct).ConfigureAwait(false); + } + + public async Task StopAsync(CancellationToken forceShutdownToken) + { + if (_runner is null) + return; + + logger.LogInformation("Shutting down background queue"); + try + { + // Wait until all the producers are done + await settings.CompletionTrigger(forceShutdownToken).ConfigureAwait(false); + } + finally + { + await _runner.Stop(null, forceShutdownToken).ConfigureAwait(false); + } + } + + public void Dispose() + { + _runner?.Dispose(); + } +} diff --git a/src/LocalPost/BackgroundQueue/ConsumeContext.cs b/src/LocalPost/BackgroundQueue/ConsumeContext.cs new file mode 100644 index 0000000..c9864e7 --- /dev/null +++ b/src/LocalPost/BackgroundQueue/ConsumeContext.cs @@ -0,0 +1,32 @@ +using System.Diagnostics; +using JetBrains.Annotations; + +namespace LocalPost.BackgroundQueue; + +[PublicAPI] +public readonly record struct ConsumeContext // Better name?.. +{ + public readonly ActivityContext? ActivityContext; + public readonly T Payload; + + internal ConsumeContext(T payload) : this(payload, Activity.Current?.Context) + { + } + + internal ConsumeContext(T payload, ActivityContext? activityContext) + { + Payload = payload; + ActivityContext = activityContext; + } + + public ConsumeContext Transform(TOut payload) => new(payload, ActivityContext); + + public static implicit operator ConsumeContext(T payload) => new(payload); + + public static implicit operator T(ConsumeContext context) => context.Payload; + + public void Deconstruct(out T payload) + { + payload = Payload; + } +} diff --git a/src/LocalPost/BackgroundQueue/DependencyInjection/BackgroundQueuesBuilder.cs b/src/LocalPost/BackgroundQueue/DependencyInjection/BackgroundQueuesBuilder.cs new file mode 100644 index 0000000..1f5b675 --- /dev/null +++ b/src/LocalPost/BackgroundQueue/DependencyInjection/BackgroundQueuesBuilder.cs @@ -0,0 +1,79 @@ +using System.Threading.Channels; +using LocalPost.DependencyInjection; +using LocalPost.Flow; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace LocalPost.BackgroundQueue.DependencyInjection; + +[PublicAPI] +public class BackgroundQueuesBuilder(IServiceCollection services) +{ + public OptionsBuilder> AddDefaultJobQueue() => AddJobQueue( + HandlerStack.For(async (job, ct) => await job(ct).ConfigureAwait(false)) + .Scoped() + .UseMessagePayload() + .Trace() + .LogExceptions() + ); + + // TODO Open later + internal OptionsBuilder> AddJobQueue( + HandlerManagerFactory> hmf) + { + services.TryAddSingleton(); + services.TryAddSingletonAlias(); + + return AddQueue(hmf); + } + + // public OptionsBuilder> AddQueue(HandlerFactory> hf) => + // AddQueue(Options.DefaultName, hf); + // + // public OptionsBuilder> AddQueue(string name, HandlerFactory> hf) => + // AddQueue(name, provider => new HandlerManager(hf(provider))); + + public OptionsBuilder> AddQueue(HandlerManagerFactory> hmf) => + AddQueue(Options.DefaultName, hmf); + + public OptionsBuilder> AddQueue(string name, HandlerManagerFactory> hmf) + { + if (!services.TryAddSingletonAlias, BackgroundQueue>(name)) + // throw new InvalidOperationException( + // $"{Reflection.FriendlyNameOf>(name)}> is already registered."); + throw new ArgumentException("Queue is already registered", nameof(name)); + + services.TryAddKeyedSingleton(name, CreateQueue); + services.AddHostedService(provider => provider.GetRequiredKeyedService>(name)); + + return QueueFor(name); + + BackgroundQueue CreateQueue(IServiceProvider provider, object? _) + { + var settings = provider.GetOptions>(name); + var channel = settings.Capacity switch + { + null => Channel.CreateUnbounded>(new UnboundedChannelOptions + { + SingleReader = settings.MaxConcurrency == 1, + SingleWriter = settings.SingleProducer, + }), + _ => Channel.CreateBounded>(new BoundedChannelOptions(settings.Capacity.Value) + { + FullMode = settings.FullMode, + SingleReader = settings.MaxConcurrency == 1, + SingleWriter = settings.SingleProducer, + }) + }; + var hm = hmf(provider); + + return new BackgroundQueue(provider.GetLoggerFor>(), settings, channel, hm); + } + } + + public OptionsBuilder> QueueFor() => + services.AddOptions>(); + + public OptionsBuilder> QueueFor(string name) => + services.AddOptions>(name); +} diff --git a/src/LocalPost/BackgroundQueue/DependencyInjection/HealthChecksBuilderEx.cs b/src/LocalPost/BackgroundQueue/DependencyInjection/HealthChecksBuilderEx.cs new file mode 100644 index 0000000..97fd455 --- /dev/null +++ b/src/LocalPost/BackgroundQueue/DependencyInjection/HealthChecksBuilderEx.cs @@ -0,0 +1,27 @@ +using LocalPost.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace LocalPost.BackgroundQueue.DependencyInjection; + +[PublicAPI] +public static class HealthChecksBuilderEx +{ + public static IHealthChecksBuilder AddBackgroundQueue(this IHealthChecksBuilder builder, + string name, HealthStatus? failureStatus = null, IEnumerable? tags = null) => + builder.Add(HealthChecks.Readiness>(name, failureStatus, tags)); + + public static IHealthChecksBuilder AddKafkaConsumers(this IHealthChecksBuilder builder, + HealthStatus? failureStatus = null, IEnumerable? tags = null) + { + var services = builder.Services + .Where(service => service is { IsKeyedService: true, ServiceType.IsGenericType: true } && + service.ServiceType.GetGenericTypeDefinition() == typeof(BackgroundQueue<>)) + .Select(service => (service.ServiceType, (string)service.ServiceKey!)); + + foreach (var (bqService, name) in services) + builder.Add(HealthChecks.Readiness(bqService, name, failureStatus, tags)); + + return builder; + } +} diff --git a/src/LocalPost/BackgroundQueue/DependencyInjection/ServiceCollectionEx.cs b/src/LocalPost/BackgroundQueue/DependencyInjection/ServiceCollectionEx.cs new file mode 100644 index 0000000..ac7c2af --- /dev/null +++ b/src/LocalPost/BackgroundQueue/DependencyInjection/ServiceCollectionEx.cs @@ -0,0 +1,15 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace LocalPost.BackgroundQueue.DependencyInjection; + +[PublicAPI] +public static class ServiceCollectionEx +{ + public static IServiceCollection AddBackgroundQueues(this IServiceCollection services, + Action configure) + { + configure(new BackgroundQueuesBuilder(services)); + + return services; + } +} diff --git a/src/LocalPost/BackgroundQueue/HandlerStackEx.cs b/src/LocalPost/BackgroundQueue/HandlerStackEx.cs new file mode 100644 index 0000000..77105bb --- /dev/null +++ b/src/LocalPost/BackgroundQueue/HandlerStackEx.cs @@ -0,0 +1,34 @@ +using System.Diagnostics; + +namespace LocalPost.BackgroundQueue; + +[PublicAPI] +public static class HandlerStackEx +{ + public static HandlerManagerFactory> UseMessagePayload(this HandlerManagerFactory hmf) => + hmf.MapHandler, T>(next => async (context, ct) => + await next(context.Payload, ct).ConfigureAwait(false)); + + public static HandlerManagerFactory> Trace(this HandlerManagerFactory> hmf) + { + var typeName = Reflection.FriendlyNameOf(); + var transactionName = $"{typeName} process"; + return hmf.TouchHandler(next => async (context, ct) => + { + using var activity = context.ActivityContext.HasValue + ? Tracing.Source.StartActivity(transactionName, ActivityKind.Consumer, + context.ActivityContext.Value) + : Tracing.Source.StartActivity(transactionName, ActivityKind.Consumer); + try + { + await next(context, ct).ConfigureAwait(false); + activity?.Success(); + } + catch (Exception e) + { + activity?.Error(e); + throw; + } + }); + } +} diff --git a/src/LocalPost/BackgroundQueue/Options.cs b/src/LocalPost/BackgroundQueue/Options.cs new file mode 100644 index 0000000..7751613 --- /dev/null +++ b/src/LocalPost/BackgroundQueue/Options.cs @@ -0,0 +1,33 @@ +using System.ComponentModel.DataAnnotations; +using System.Threading.Channels; + +namespace LocalPost.BackgroundQueue; + +public sealed record QueueOptions +{ + /// + /// How many messages to process concurrently. Default is 10. + /// + [Required] + [Range(1, ushort.MaxValue)] + public ushort MaxConcurrency { get; set; } = 10; + + public bool ProcessLeftovers { get; set; } = true; + + [Range(1, int.MaxValue)] + public int? Capacity { get; set; } = 1000; + + /// + /// How to handle new messages when the underlying channel is full. Default is to drop the oldest message + /// (to not block the producer). + /// + public BoundedChannelFullMode FullMode { get; set; } = BoundedChannelFullMode.DropOldest; + + public bool SingleProducer { get; set; } = false; + + /// + /// How long to wait before closing the queue (channel) on app shutdown. Default is 1 second. + /// + public Func CompletionTrigger { get; set; } = ct => Task.Delay(1000, ct); + // public ushort CompletionDelay { get; set; } = 1_000; // Milliseconds +} diff --git a/src/LocalPost/BackgroundQueue/Tracing.cs b/src/LocalPost/BackgroundQueue/Tracing.cs new file mode 100644 index 0000000..03e720e --- /dev/null +++ b/src/LocalPost/BackgroundQueue/Tracing.cs @@ -0,0 +1,18 @@ +using System.Diagnostics; +using System.Reflection; + +namespace LocalPost.BackgroundQueue; + +internal static class Tracing +{ + public static readonly ActivitySource Source; + + public static bool IsEnabled => Source.HasListeners(); + + static Tracing() + { + var assembly = Assembly.GetExecutingAssembly(); + var version = assembly.GetName().Version?.ToString() ?? "0.0.0"; + Source = new ActivitySource("LocalPost.BackgroundQueue", version); + } +} diff --git a/src/LocalPost/BackgroundQueueConsumer.cs b/src/LocalPost/BackgroundQueueConsumer.cs deleted file mode 100644 index 6b645f9..0000000 --- a/src/LocalPost/BackgroundQueueConsumer.cs +++ /dev/null @@ -1,108 +0,0 @@ -using System.Threading.Channels; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; - -namespace LocalPost; - -internal sealed class BackgroundQueueConsumer : BackgroundService -{ - private readonly ILogger> _logger; - private readonly string _name; - private readonly IServiceScopeFactory _scopeFactory; - - private readonly IAsyncEnumerable _reader; - private readonly IExecutor _executor; - private readonly Func> _handlerFactory; - - public BackgroundQueueConsumer(string name, - ILogger> logger, IServiceScopeFactory scopeFactory, - IExecutor executor, IAsyncEnumerable reader, Func> handlerFactory) - { - _logger = logger; - _name = name; - _scopeFactory = scopeFactory; - _reader = reader; - _executor = executor; - _handlerFactory = handlerFactory; - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - _logger.LogInformation("Starting {Name} background queue...", _name); - - try - { - await foreach (var message in _reader.WithCancellation(stoppingToken)) - await _executor.StartAsync(() => Process(message, stoppingToken), stoppingToken); - } - catch (OperationCanceledException e) when (e.CancellationToken == stoppingToken) - { - // The rest of the queue will be processed in StopAsync() below - _logger.LogInformation("Application exit has been requested, stopping {Name} background queue...", _name); - } - catch (ChannelClosedException e) - { - _logger.LogWarning(e, "{Name} queue has been closed, stop listening", _name); - - // The rest of the queue will be processed in StopAsync() below - } - catch (Exception e) - { - // Custom error handler?.. - _logger.LogCritical(e, "Unhandled exception, stop listening"); - } - } - - public override async Task StopAsync(CancellationToken forceExitToken) - { - await base.StopAsync(forceExitToken); - - var enumerator = _reader.GetAsyncEnumerator(forceExitToken); - var move = enumerator.MoveNextAsync(); - var completed = false; - do - { - // Suck out all the _available_ messages - while (move.IsCompleted) - { - completed = !await move; - if (completed) - break; - - await _executor.StartAsync(() => Process(enumerator.Current, forceExitToken), forceExitToken); - - move = enumerator.MoveNextAsync(); - } - - if (_executor.IsEmpty) - // It means that nothing has been started (no messages read), so we are finally done - break; - - // Wait until all currently running tasks are finished - await _executor.WaitAsync(forceExitToken); - } while (!completed); - } - - private async Task Process(T message, CancellationToken ct) - { - using var scope = _scopeFactory.CreateScope(); - - // Make it specific for this queue somehow?.. - var handler = _handlerFactory(scope.ServiceProvider); - - try - { - // Await the handler, to keep the container scope alive - await handler(message, ct); - } - catch (OperationCanceledException e) when (e.CancellationToken == ct) - { - throw; - } - catch (Exception e) - { - _logger.LogError(e, "{Name} queue: unhandled exception while processing a message", _name); - } - } -} diff --git a/src/LocalPost/BatchBuilder.cs b/src/LocalPost/BatchBuilder.cs deleted file mode 100644 index 2bc3e12..0000000 --- a/src/LocalPost/BatchBuilder.cs +++ /dev/null @@ -1,67 +0,0 @@ -using Nito.AsyncEx; - -namespace LocalPost; - -internal delegate IBatchBuilder BatchBuilderFactory(); - -internal interface IBatchBuilder : IDisposable -{ - CancellationToken TimeWindow { get; } - Task TimeWindowTrigger { get; } - - bool IsEmpty { get; } - - bool TryAdd(T entry); - - TBatch Build(); -} - -internal abstract class BatchBuilder : IBatchBuilder -{ - private readonly CancellationTokenSource _timeWindow; - private readonly CancellationTokenTaskSource _timeWindowTrigger; - - protected BatchBuilder(TimeSpan timeWindow) - { - _timeWindow = new CancellationTokenSource(timeWindow); - _timeWindowTrigger = new CancellationTokenTaskSource(_timeWindow.Token); - } - - public CancellationToken TimeWindow => _timeWindow.Token; - public Task TimeWindowTrigger => _timeWindowTrigger.Task; - public abstract bool IsEmpty { get; } - - public abstract bool TryAdd(T entry); - public abstract TBatch Build(); - - public void Dispose() - { - _timeWindow.Dispose(); - _timeWindowTrigger.Dispose(); - } -} - -internal sealed class BoundedBatchBuilder : BatchBuilder> -{ - private readonly int _max; - private readonly List _batch = new(); - - public BoundedBatchBuilder(int max, TimeSpan timeWindow) : base(timeWindow) - { - _max = max; - } - - public override bool IsEmpty => _batch.Count == 0; - - public override bool TryAdd(T entry) - { - if (_batch.Count >= _max) - return false; - - _batch.Add(entry); - - return true; - } - - public override IReadOnlyList Build() => _batch; -} diff --git a/src/LocalPost/BatchingAsyncEnumerable.cs b/src/LocalPost/BatchingAsyncEnumerable.cs deleted file mode 100644 index 071d0b0..0000000 --- a/src/LocalPost/BatchingAsyncEnumerable.cs +++ /dev/null @@ -1,96 +0,0 @@ -namespace LocalPost; - -internal sealed class BatchingAsyncEnumerable : IAsyncEnumerable -{ - private readonly IAsyncEnumerable _reader; - private readonly BatchBuilderFactory _factory; - - public BatchingAsyncEnumerable(IAsyncEnumerable source, BatchBuilderFactory factory) - { - _reader = source; - _factory = factory; - - } - - private void HandleSkipped(T item) - { - } - - public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken ct = default) - { - var batch = _factory(); - IBatchBuilder ShiftBatch() - { - batch.Dispose(); - return batch = _factory(); - } - - try - { - var source = _reader.GetAsyncEnumerator(ct); - var completed = false; - var waitTrigger = source.MoveNextAsync(); - Task? waitTask = null; - while (!completed) - { - var shift = false; - try - { - if (waitTask is null && waitTrigger.IsCompleted) - completed = !await waitTrigger; - else - { - waitTask ??= waitTrigger.AsTask(); - // The same as Task.WaitAsync(batch.TimeWindow), but saves some allocations - completed = !await (await Task.WhenAny(waitTask, batch.TimeWindowTrigger)); - waitTask = null; - } - - if (completed) - continue; - } - catch (OperationCanceledException e) when (e.CancellationToken == batch.TimeWindow) - { - shift = true; - } - catch (OperationCanceledException) // User cancellation - { - completed = true; - continue; - } - - if (shift) - { - if (!batch.IsEmpty) - yield return batch.Build(); - - ShiftBatch(); - continue; - } - - if (!batch.TryAdd(source.Current)) - if (!batch.IsEmpty) - { - // Flush the current buffer and start a fresh one - yield return batch.Build(); - if (!ShiftBatch().TryAdd(source.Current)) - HandleSkipped(source.Current); // Even an empty buffer cannot fit it... - } - else - HandleSkipped(source.Current); // Even an empty buffer cannot fit it... - - waitTrigger = source.MoveNextAsync(); - } - - // Flush on completion or error... - if (!batch.IsEmpty) - yield return batch.Build(); - - ct.ThrowIfCancellationRequested(); - } - finally - { - batch.Dispose(); - } - } -} diff --git a/src/LocalPost/ChannelReaderExtensions.cs b/src/LocalPost/ChannelReaderExtensions.cs deleted file mode 100644 index 2f364d1..0000000 --- a/src/LocalPost/ChannelReaderExtensions.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System.Runtime.CompilerServices; -using System.Threading.Channels; - -namespace LocalPost; - -internal static class ChannelReaderExtensions -{ - // netstandard2.0 does not contain this overload, it's available only from netstandard2.1 (.NET Core 3.0) - public static async IAsyncEnumerable ReadAllAsync(this ChannelReader reader, - [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - while (await reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) - while (reader.TryRead(out var item)) - yield return item; - } -} diff --git a/src/LocalPost/ConcurrentSet.cs b/src/LocalPost/ConcurrentSet.cs deleted file mode 100644 index e52964d..0000000 --- a/src/LocalPost/ConcurrentSet.cs +++ /dev/null @@ -1,66 +0,0 @@ -using System.Collections; -using System.Collections.Immutable; -using Nito.AsyncEx; - -namespace LocalPost; - -internal sealed class ConcurrentSet : IEnumerable, IDisposable -{ - private readonly object _modificationLock = new(); - private ImmutableHashSet? _elements; - private CancellationTokenSource _modificationTriggerSource = new(); - private CancellationTokenTaskSource _modificationTriggerTaskSource; - - public ConcurrentSet(IEnumerable sources) - { - _modificationTriggerTaskSource = new CancellationTokenTaskSource(_modificationTriggerSource.Token); - _elements = sources.ToImmutableHashSet(); - } - - public ImmutableHashSet Elements => _elements ?? throw new ObjectDisposedException(nameof(ConcurrentSet)); - - public CancellationToken ModificationToken => _modificationTriggerSource.Token; - - public Task Modification => _modificationTriggerTaskSource.Task; - - private ImmutableHashSet ChangeSources(Func, ImmutableHashSet> change) - { - ImmutableHashSet changedSources; - CancellationTokenSource trigger; - CancellationTokenTaskSource triggerTask; - lock (_modificationLock) - { - changedSources = change(Elements); - if (changedSources == _elements) - return _elements; // Nothing has changed - - _elements = changedSources; - trigger = _modificationTriggerSource; - triggerTask = _modificationTriggerTaskSource; - _modificationTriggerSource = new CancellationTokenSource(); - _modificationTriggerTaskSource = new CancellationTokenTaskSource(_modificationTriggerSource.Token); - } - - trigger.Cancel(); // Notify about the modification - - triggerTask.Dispose(); - trigger.Dispose(); - - return changedSources; - } - - public ImmutableHashSet Add(T source) => ChangeSources(sources => sources.Add(source)); - - public ImmutableHashSet Remove(T source) => ChangeSources(sources => sources.Remove(source)); - - public IEnumerator GetEnumerator() => Elements.GetEnumerator(); - - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - - public void Dispose() - { - _modificationTriggerSource.Dispose(); - _modificationTriggerTaskSource.Dispose(); - _elements = null; - } -} diff --git a/src/LocalPost/ConcurrentTasksList.cs b/src/LocalPost/ConcurrentTasksList.cs deleted file mode 100644 index 29d3ada..0000000 --- a/src/LocalPost/ConcurrentTasksList.cs +++ /dev/null @@ -1,42 +0,0 @@ -using Nito.AsyncEx; - -namespace LocalPost; - -internal sealed class ConcurrentTasksList -{ - private readonly object _tasksLock = new(); - private readonly List _tasks; - - public ConcurrentTasksList(int capacityHint) - { - _tasks = new List(capacityHint); - } - - public int Count => _tasks.Count; - - private Task WhenAny() - { - lock (_tasksLock) - return Task.WhenAny(_tasks); - } - - private void Remove(Task item) - { - lock (_tasksLock) - _tasks.Remove(item); - } - - public void CleanupCompleted() - { - lock (_tasksLock) - _tasks.RemoveAll(task => task.IsCompleted); - } - - public void Track(Task item) - { - lock (_tasksLock) - _tasks.Add(item); - } - - public async Task WaitForCompleted(CancellationToken ct) => Remove(await WhenAny().WaitAsync(ct)); -} diff --git a/src/LocalPost/DependencyInjection/ConfigurationExtensions.cs b/src/LocalPost/DependencyInjection/ConfigurationExtensions.cs deleted file mode 100644 index b3925d2..0000000 --- a/src/LocalPost/DependencyInjection/ConfigurationExtensions.cs +++ /dev/null @@ -1,15 +0,0 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; - -namespace LocalPost.DependencyInjection; - -// TODO Open to public later -internal static class ConfigurationExtensions -{ - public static OptionsBuilder AddBackgroundQueueOptions(this IServiceCollection services) => - services.AddOptions(Reflection.FriendlyNameOf>()); - - public static OptionsBuilder AddBackgroundQueueOptions(this IServiceCollection services, string name) => - services.AddOptions(name); -} diff --git a/src/LocalPost/DependencyInjection/CustomQueueRegistrationExtensions.cs b/src/LocalPost/DependencyInjection/CustomQueueRegistrationExtensions.cs deleted file mode 100644 index 0378885..0000000 --- a/src/LocalPost/DependencyInjection/CustomQueueRegistrationExtensions.cs +++ /dev/null @@ -1,37 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; - -namespace LocalPost.DependencyInjection; - -public static class CustomQueueRegistrationExtensions -{ - // TReader & THandler have to be registered by the user - public static OptionsBuilder AddCustomBackgroundQueue( - this IServiceCollection services) where TReader : IAsyncEnumerable where THandler : IMessageHandler => - services.AddCustomBackgroundQueue(provider => provider.GetRequiredService().Process); - - // TReader has to be registered by the user - public static OptionsBuilder AddCustomBackgroundQueue(this IServiceCollection services, - Func> handlerFactory) - where TReader : IAsyncEnumerable => - services.AddCustomBackgroundQueue(Reflection.FriendlyNameOf(), - provider => provider.GetRequiredService(), handlerFactory); - - public static OptionsBuilder AddCustomBackgroundQueue(this IServiceCollection services, string name, - Func> readerFactory, - Func> handlerFactory) - { - // TODO Try...() version of this one, to be gentle with multiple registrations of the same queue?.. - services.AddHostedService(provider => - { - var executor = ActivatorUtilities.CreateInstance(provider, name); - - return ActivatorUtilities.CreateInstance>(provider, name, - readerFactory(provider), executor, handlerFactory); - }); - - // TODO Health check, metrics - - return services.AddOptions(name);; - } -} diff --git a/src/LocalPost/DependencyInjection/HealthChecks.cs b/src/LocalPost/DependencyInjection/HealthChecks.cs new file mode 100644 index 0000000..07ada20 --- /dev/null +++ b/src/LocalPost/DependencyInjection/HealthChecks.cs @@ -0,0 +1,55 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; + +namespace LocalPost.DependencyInjection; + +internal interface IHealthAwareService +{ + IHealthCheck ReadinessCheck { get; } +} + +[PublicAPI] +public static partial class ServiceCollectionEx +{ + public static IServiceCollection AddAppHealthSupervisor(this IServiceCollection services, + IEnumerable? tags = null) + { + services.AddSingleton(provider => new AppHealthSupervisor( + provider.GetLoggerFor(), + provider.GetRequiredService(), + provider.GetRequiredService()) + { + Tags = tags?.ToImmutableHashSet() ?? ImmutableHashSet.Empty + }); + + services.AddHostedService(); + + return services; + } +} + +internal static partial class HealthChecks +{ + private sealed class LambdaHealthCheck(Func check) : IHealthCheck + { + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken ct = default) => + Task.FromResult(check()); + } + + public static IHealthCheck From(Func check) => + new LambdaHealthCheck(check); + + public static HealthCheckRegistration Readiness(string name, + HealthStatus? failureStatus = null, IEnumerable? tags = null) + where T : IHealthAwareService => + Readiness(typeof(T), name, failureStatus, tags); + + public static HealthCheckRegistration Readiness(Type bqService, string name, + HealthStatus? failureStatus = null, IEnumerable? tags = null) => + new(name, // Can be overwritten later + provider => ((IHealthAwareService)provider.GetRequiredKeyedService(bqService, name)).ReadinessCheck, + failureStatus, // Can be overwritten later + tags); +} diff --git a/src/LocalPost/DependencyInjection/JobQueueRegistrationExtensions.cs b/src/LocalPost/DependencyInjection/JobQueueRegistrationExtensions.cs deleted file mode 100644 index a9eb008..0000000 --- a/src/LocalPost/DependencyInjection/JobQueueRegistrationExtensions.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; - -namespace LocalPost.DependencyInjection; - -public static class JobQueueRegistrationExtensions -{ - public static OptionsBuilder AddBackgroundJobQueue(this IServiceCollection services) - { - services.TryAddSingleton(); - services.TryAddSingleton(provider => provider.GetRequiredService()); - - return services.AddCustomBackgroundQueue(_ => (job, ct) => job(ct)); - } -} diff --git a/src/LocalPost/DependencyInjection/QueueRegistrationExtensions.cs b/src/LocalPost/DependencyInjection/QueueRegistrationExtensions.cs deleted file mode 100644 index 567a90a..0000000 --- a/src/LocalPost/DependencyInjection/QueueRegistrationExtensions.cs +++ /dev/null @@ -1,23 +0,0 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; - -namespace LocalPost.DependencyInjection; - -public static class QueueRegistrationExtensions -{ - // THandler has to be registered by the user - public static OptionsBuilder AddBackgroundQueue(this IServiceCollection services) - where THandler : IMessageHandler => - services.AddBackgroundQueue(provider => provider.GetRequiredService().Process); - - public static OptionsBuilder AddBackgroundQueue(this IServiceCollection services, - Func> handlerFactory) - { - services.TryAddSingleton>(); - services.TryAddSingleton>(provider => provider.GetRequiredService>()); - - return services.AddCustomBackgroundQueue>(handlerFactory); - } -} diff --git a/src/LocalPost/DependencyInjection/ServiceCollectionTools.cs b/src/LocalPost/DependencyInjection/ServiceCollectionTools.cs new file mode 100644 index 0000000..644d1c7 --- /dev/null +++ b/src/LocalPost/DependencyInjection/ServiceCollectionTools.cs @@ -0,0 +1,72 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace LocalPost.DependencyInjection; + +internal static class ServiceCollectionTools +{ + public static IEnumerable GetKeysFor(this IServiceCollection services) => services + // See https://github.com/dotnet/runtime/issues/95789#issuecomment-2274223124 + .Where(service => service.IsKeyedService && service.ServiceType == typeof(T)) + .Select(service => service.ServiceKey); + + public static bool TryAddKeyedSingleton(this IServiceCollection services, object key, + Func factory) + // where TService : class, INamedService => + where TService : class => + services.TryAdd(ServiceDescriptor.KeyedSingleton(key, factory)); + + public static bool TryAddSingleton(this IServiceCollection services) where TService : class => + services.TryAdd(ServiceDescriptor.Singleton()); + + public static bool TryAddSingleton(this IServiceCollection services, + Func factory) + where TService : class => + services.TryAdd(ServiceDescriptor.Singleton(factory)); + + // "If binary compatibility were not a problem, then the TryAdd methods could return bool" + // from https://github.com/dotnet/runtime/issues/45114#issuecomment-733807639 + // See also: https://github.com/dotnet/runtime/issues/44728#issuecomment-831413792 + public static bool TryAdd(this IServiceCollection services, ServiceDescriptor descriptor) + { + if (services.Any(service => IsEqual(service, descriptor))) + return false; + + services.Add(descriptor); + return true; + + static bool IsEqual(ServiceDescriptor a, ServiceDescriptor b) + { + var equal = a.ServiceType == b.ServiceType && a.IsKeyedService == b.IsKeyedService; + + return equal && a.IsKeyedService ? a.ServiceKey == b.ServiceKey : equal; + } + } + + public static IServiceCollection AddSingletonAlias(this IServiceCollection services) + where TService : class + where TImplementation : class, TService => + services.AddSingleton(provider => provider.GetRequiredService()); + + public static IServiceCollection AddSingletonAlias(this IServiceCollection services, object key) + where TAlias : class + where TService : class, TAlias => + services.AddKeyedSingleton(key, (provider, _) => provider.GetRequiredKeyedService(key)); + + public static bool TryAddSingletonAlias(this IServiceCollection services) + where TAlias : class + where TService : class, TAlias => + services.TryAddSingleton(provider => provider.GetRequiredService()); + + public static bool TryAddSingletonAlias(this IServiceCollection services, object key) + where TAlias : class + where TService : class, TAlias + { + var added = services.TryAddKeyedSingleton(key, (provider, _) => + provider.GetRequiredKeyedService(key)); + if (added && key is string name && name == Options.DefaultName) + services.TryAddSingleton(provider => + provider.GetRequiredKeyedService(Options.DefaultName)); + return added; + } +} diff --git a/src/LocalPost/DependencyInjection/ServiceProviderLookups.cs b/src/LocalPost/DependencyInjection/ServiceProviderLookups.cs new file mode 100644 index 0000000..de178c1 --- /dev/null +++ b/src/LocalPost/DependencyInjection/ServiceProviderLookups.cs @@ -0,0 +1,16 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace LocalPost.DependencyInjection; + +internal static class ServiceProviderLookups +{ + public static T GetOptions(this IServiceProvider provider) where T : class => + provider.GetRequiredService>().Value; + + public static T GetOptions(this IServiceProvider provider, string name) where T : class => + provider.GetRequiredService>().Get(name); + + public static ILogger GetLoggerFor(this IServiceProvider provider) => + provider.GetRequiredService>(); +} diff --git a/src/LocalPost/Executor.cs b/src/LocalPost/Executor.cs deleted file mode 100644 index a8b3f2e..0000000 --- a/src/LocalPost/Executor.cs +++ /dev/null @@ -1,57 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using Microsoft.Extensions.Options; - -namespace LocalPost; - -internal interface IExecutor -{ - public bool IsEmpty { get; } - - // Start only when there is a capacity - ValueTask StartAsync(Func itemProcessor, CancellationToken ct); - - // Wait for all active tasks to finish... - ValueTask WaitAsync(CancellationToken ct); -} - -internal sealed class BoundedExecutor : IExecutor -{ - private readonly ConcurrentTasksList _tasks; - - [ExcludeFromCodeCoverage] - public BoundedExecutor(string name, IOptionsMonitor options) : this(options.Get(name).MaxConcurrency) - { - } - - public BoundedExecutor(ushort maxConcurrency = ushort.MaxValue) - { - MaxConcurrency = maxConcurrency; - _tasks = new ConcurrentTasksList(MaxConcurrency); - } - - public ushort MaxConcurrency { get; } - - public bool IsEmpty => _tasks.Count == 0; - - private async ValueTask WaitForCapacityAsync(CancellationToken ct) - { - _tasks.CleanupCompleted(); - while (_tasks.Count >= MaxConcurrency) - await _tasks.WaitForCompleted(ct); - } - - public async ValueTask StartAsync(Func itemProcessor, CancellationToken ct) - { - if (_tasks.Count >= MaxConcurrency) - await WaitForCapacityAsync(ct); - - _tasks.Track(itemProcessor()); - } - - public async ValueTask WaitAsync(CancellationToken ct) - { - _tasks.CleanupCompleted(); - while (!IsEmpty) - await _tasks.WaitForCompleted(ct); - } -} diff --git a/src/LocalPost/Flow/HandlerStackEx.cs b/src/LocalPost/Flow/HandlerStackEx.cs new file mode 100644 index 0000000..fd027cc --- /dev/null +++ b/src/LocalPost/Flow/HandlerStackEx.cs @@ -0,0 +1,221 @@ +using System.Collections.Immutable; +using System.Threading.Channels; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace LocalPost.Flow; + + +[PublicAPI] +public static partial class HandlerStackEx +{ + public static HandlerManagerFactory Buffer(this HandlerManagerFactory hmf, + int capacity, int consumers = 1, bool singleProducer = false) + { + var channel = Channel.CreateBounded(new BoundedChannelOptions(capacity) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = consumers == 1, + SingleWriter = singleProducer, + }); + + return hmf.Buffer(channel, consumers); + } + + private static HandlerManagerFactory Buffer(this HandlerManagerFactory hmf, + Channel channel, int consumers = 1) => provider => + new BufferHandlerManager(hmf(provider), channel, consumers); + + public static HandlerManagerFactory Batch(this HandlerManagerFactory> hmf, + int size, TimeSpan window, + int capacity = 1, bool singleProducer = false) => provider => + { + var channel = Channel.CreateBounded(new BoundedChannelOptions(capacity) + { + FullMode = BoundedChannelFullMode.Wait, + SingleReader = true, + SingleWriter = singleProducer, + }); + var next = hmf(provider); + return new BatchHandlerManager(next, channel, size, window); + }; +} + +internal sealed class BufferHandlerManager(IHandlerManager next, Channel channel, + int consumers) : IHandlerManager +{ + private ChannelRunner? _runner; + + private ChannelRunner CreateRunner(Handler handler) + { + return new ChannelRunner(channel, Consume, next) { Consumers = consumers }; + + async Task Consume(CancellationToken execToken) + { + await foreach (var message in channel.Reader.ReadAllAsync(execToken).ConfigureAwait(false)) + await handler(message, CancellationToken.None).ConfigureAwait(false); + } + } + + public async ValueTask> Start(CancellationToken ct) + { + var handler = await next.Start(ct).ConfigureAwait(false); + _runner = CreateRunner(handler); + await _runner.Start(ct).ConfigureAwait(false); + return channel.Writer.WriteAsync; + } + + public async ValueTask Stop(Exception? error, CancellationToken ct) + { + if (_runner is not null) + await _runner.Stop(error, ct).ConfigureAwait(false); + await next.Stop(error, ct).ConfigureAwait(false); + } +} + +internal sealed class BatchHandlerManager(IHandlerManager> next, Channel channel, + int size, TimeSpan window) : IHandlerManager +{ + private ChannelRunner>? _runner; + + private ChannelRunner> CreateRunner(Handler> handler) + { + return new ChannelRunner>(channel, Consume, next) { Consumers = 1 }; + + async Task Consume(CancellationToken execToken) + { + var completed = false; + while (!completed) + { + var batch = new List(size); + using var timeWindowCts = CancellationTokenSource.CreateLinkedTokenSource(execToken); + timeWindowCts.CancelAfter(window); + try + { + while (batch.Count < size) + { + var item = await channel.Reader.ReadAsync(timeWindowCts.Token).ConfigureAwait(false); + batch.Add(item); + } + } + catch (OperationCanceledException) when (!execToken.IsCancellationRequested) + { + // Batch window is closed + } + catch (Exception) // execToken.IsCancellationRequested + ChannelClosedException + { + completed = true; + } + + if (batch.Count == 0) + continue; + + await handler(batch, CancellationToken.None).ConfigureAwait(false); + } + } + } + + public async ValueTask> Start(CancellationToken ct) + { + var handler = await next.Start(ct).ConfigureAwait(false); + _runner = CreateRunner(handler); + await _runner.Start(ct).ConfigureAwait(false); + return channel.Writer.WriteAsync; + } + + public async ValueTask Stop(Exception? error, CancellationToken ct) + { + if (_runner is not null) + await _runner.Stop(error, ct).ConfigureAwait(false); + await next.Stop(error, ct).ConfigureAwait(false); + } +} + +internal sealed class ChannelRunner(Channel channel, + Func consumer, IHandlerManager handler) : IDisposable +{ + public HealthCheckResult Ready => (_execTokenSource, _exec, _execException) switch + { + (null, _, _) => HealthCheckResult.Unhealthy("Not started"), + (_, { IsCompleted: true }, _) => HealthCheckResult.Unhealthy("Stopped"), + (not null, null, _) => HealthCheckResult.Degraded("Starting"), + (not null, not null, null) => HealthCheckResult.Healthy("Running"), + (_, _, not null) => HealthCheckResult.Unhealthy(null, _execException), + }; + + public PositiveInt Consumers { get; init; } = 1; + public bool ProcessLeftovers { get; init; } = true; + + private CancellationTokenSource? _execTokenSource; + private Task? _exec; + private Exception? _execException; + + private CancellationToken _completionToken = CancellationToken.None; + + public async ValueTask Start(CancellationToken ct) + { + if (_execTokenSource is not null) + throw new InvalidOperationException("Already started"); + + var execTokenSource = _execTokenSource = new CancellationTokenSource(); + + await handler.Start(ct).ConfigureAwait(false); + + _exec = Run(execTokenSource.Token); + } + + private async Task Run(CancellationToken execToken) + { + var exec = Consumers.Value switch + { + 1 => RunConsumer(execToken), + _ => Task.WhenAll(Enumerable.Range(0, Consumers).Select(_ => RunConsumer(execToken))) + }; + await exec.ConfigureAwait(false); + + await handler.Stop(_execException, _completionToken).ConfigureAwait(false); + } + + private async Task RunConsumer(CancellationToken execToken) + { + try + { + await consumer(execToken).ConfigureAwait(false); + Close(); + } + catch (OperationCanceledException e) when (e.CancellationToken == execToken) + { + // OK, fine + } + catch (ChannelClosedException) + { + // OK, fine + } + catch (Exception e) + { + Close(e); + } + } + + public async ValueTask Stop(Exception? e, CancellationToken ct) + { + _completionToken = ct; + Close(e); + if (_exec is not null) + await _exec.ConfigureAwait(false); + } + + public void Dispose() + { + _execTokenSource?.Dispose(); + _exec?.Dispose(); + } + + private void Close(Exception? e = null) + { + if (!channel.Writer.TryComplete(e)) + return; + _execException ??= e; + if (!ProcessLeftovers) + _execTokenSource?.Cancel(); + } +} diff --git a/src/LocalPost/Handler.cs b/src/LocalPost/Handler.cs new file mode 100644 index 0000000..517362b --- /dev/null +++ b/src/LocalPost/Handler.cs @@ -0,0 +1,50 @@ +namespace LocalPost; + +public delegate ValueTask Handler(T context, CancellationToken ct); + +public delegate Handler HandlerFactory(IServiceProvider provider); + +public delegate Handler HandlerMiddleware(Handler next); + +// Too narrow use case +// public delegate HandlerMiddleware HandlerMiddlewareFactory(IServiceProvider provider); + +// Even more narrow use case, confuses more than helps +// public delegate HandlerFactory HandlerFactoryMiddleware(HandlerFactory hf); + +public interface IHandler +{ + ValueTask InvokeAsync(TOut payload, CancellationToken ct); +} + + + +public delegate IHandlerManager HandlerManagerFactory(IServiceProvider provider); + +public delegate IHandlerManager HandlerManagerMiddleware(IHandlerManager next); + +public interface IHandlerManager +{ + ValueTask> Start(CancellationToken ct); + + ValueTask Stop(Exception? error, CancellationToken ct); +} + +internal sealed class HandlerManager(Handler handler) : IHandlerManager +{ + public ValueTask> Start(CancellationToken ct) => ValueTask.FromResult(handler); + + public ValueTask Stop(Exception? error, CancellationToken ct) => ValueTask.CompletedTask; +} + +internal sealed class HandlerDecorator( + IHandlerManager next, HandlerMiddleware middleware) : IHandlerManager +{ + public async ValueTask> Start(CancellationToken ct) + { + var nextHandler = await next.Start(ct).ConfigureAwait(false); + return middleware(nextHandler); + } + + public ValueTask Stop(Exception? error, CancellationToken ct) => next.Stop(error, ct); +} diff --git a/src/LocalPost/HandlerStack.cs b/src/LocalPost/HandlerStack.cs new file mode 100644 index 0000000..487777a --- /dev/null +++ b/src/LocalPost/HandlerStack.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace LocalPost; + +// [PublicAPI] +// public static class HandlerStack +// { +// public static readonly HandlerFactory Empty = _ => (_, _) => default; +// } + +// [PublicAPI] +// public static class HandlerStack +// { +// public static HandlerFactory For(Action syncHandler) => For((payload, _) => +// { +// syncHandler(payload); +// return ValueTask.CompletedTask; +// }); +// +// public static HandlerFactory For(Handler handler) => _ => handler; +// +// public static HandlerFactory From() where THandler : IHandler => +// provider => provider.GetRequiredService().InvokeAsync; +// } + +[PublicAPI] +public static class HandlerStack +{ + public static HandlerManagerFactory For(Action syncHandler) => For((payload, _) => + { + syncHandler(payload); + return ValueTask.CompletedTask; + }); + + public static HandlerManagerFactory For(Handler handler) => _ => new HandlerManager(handler); + + public static HandlerManagerFactory From() where THandler : IHandler => + provider => new HandlerManager(provider.GetRequiredService().InvokeAsync); +} diff --git a/src/LocalPost/HandlerStackOps.cs b/src/LocalPost/HandlerStackOps.cs new file mode 100644 index 0000000..6711942 --- /dev/null +++ b/src/LocalPost/HandlerStackOps.cs @@ -0,0 +1,29 @@ +namespace LocalPost; + + +[PublicAPI] +public static partial class HandlerStackOps +{ + public static IHandlerManager Map(this IHandlerManager hm, + HandlerMiddleware middleware) => new HandlerDecorator(hm, middleware); + + public static IHandlerManager Touch(this IHandlerManager hm, + HandlerMiddleware middleware) => hm.Map(middleware); + + public static HandlerManagerFactory Map(this HandlerManagerFactory hmf, + HandlerManagerMiddleware middleware) => provider => + { + var handler = hmf(provider); + return middleware(handler); + }; + + public static HandlerManagerFactory Touch(this HandlerManagerFactory hf, + HandlerManagerMiddleware middleware) => hf.Map(middleware); + + public static HandlerManagerFactory MapHandler(this HandlerManagerFactory hmf, + HandlerMiddleware middleware) => + hmf.Map(next => next.Map(middleware)); + + public static HandlerManagerFactory TouchHandler(this HandlerManagerFactory hmf, + HandlerMiddleware middleware) => hmf.MapHandler(middleware); +} diff --git a/src/LocalPost/LocalPost.csproj b/src/LocalPost/LocalPost.csproj index ceabac6..3e90087 100644 --- a/src/LocalPost/LocalPost.csproj +++ b/src/LocalPost/LocalPost.csproj @@ -1,23 +1,15 @@ - netstandard2.0 + net6;net8 LocalPost - true - - false - LocalPost - background;task;queue;coravel;hangfire + Local (in-process) background queue. - Alexey Shokov + background;task;queue;coravel;hangfire README.md - MIT - https://github.com/alexeyshockov/LocalPost - git - true @@ -25,42 +17,42 @@ - - - true - - - - true - true - true - true - snupkg + + + + + + + + + + + + + + + + - - true - + + + + + - - - - - - - - - - - + + + + - <_Parameter1>$(MSBuildProjectName).SnsPublisher + <_Parameter1>$(MSBuildProjectName).SqsConsumer - <_Parameter1>$(MSBuildProjectName).SqsConsumer + <_Parameter1>$(MSBuildProjectName).KafkaConsumer diff --git a/src/LocalPost/Middlewares.cs b/src/LocalPost/Middlewares.cs new file mode 100644 index 0000000..180b11e --- /dev/null +++ b/src/LocalPost/Middlewares.cs @@ -0,0 +1,120 @@ +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace LocalPost; + +public static partial class Middlewares +{ + public static HandlerManagerFactory DecodeString(this HandlerManagerFactory hmf) => + DecodeString(hmf, Encoding.UTF8); + + public static HandlerManagerFactory DecodeString(this HandlerManagerFactory hmf, + Encoding encoding) => hmf.MapHandler(next => + async (payload, ct) => + { + var s = encoding.GetString(payload); + await next(s, ct).ConfigureAwait(false); + }); + + /// + /// Handle exceptions and log them, to not break the consumer loop. + /// + /// Handler factory to wrap. + /// Handler's payload type. + /// Wrapped handler factory. + public static HandlerManagerFactory LogExceptions(this HandlerManagerFactory hmf) => provider => + { + var logger = provider.GetRequiredService>(); + return hmf(provider).Touch(next => async (context, ct) => + { + try + { + await next(context, ct).ConfigureAwait(false); + } + catch (OperationCanceledException e) when (e.CancellationToken == ct) + { + throw; + } + catch (Exception e) + { + logger.LogError(e, "Unhandled exception while processing a message"); + } + } + ); + }; + + /// + /// Create a DI scope for every message and resolve the handler from it. + /// + /// Handler factory to wrap. + /// Handler's payload type. + /// Wrapped handler factory. + public static HandlerManagerFactory Scoped(this HandlerManagerFactory hmf) => provider => + new ScopedHandlerManager(provider.GetRequiredService(), hmf); + + /// + /// Shutdown the whole app on error. + /// + /// Handler factory to wrap. + /// Process exit code. + /// Handler's payload type. + /// Wrapped handler factory. + public static HandlerManagerFactory ShutdownOnError(this HandlerManagerFactory hmf, int exitCode = 1) => + provider => + { + var appLifetime = provider.GetRequiredService(); + return hmf(provider).Touch(next => async (context, ct) => + { + try + { + await next(context, ct).ConfigureAwait(false); + } + catch (OperationCanceledException e) when (e.CancellationToken == ct) + { + throw; + } + catch + { + appLifetime.StopApplication(); + Environment.ExitCode = exitCode; + } + }); + }; +} + + +internal sealed class ScopedHandlerManager(IServiceScopeFactory sf, HandlerManagerFactory hmf) + : IHandlerManager +{ + private async ValueTask Handle(T payload, CancellationToken ct) + { + // See https://andrewlock.net/exploring-dotnet-6-part-10-new-dependency-injection-features-in-dotnet-6/#handling-iasyncdisposable-services-with-iservicescope + // And also https://devblogs.microsoft.com/dotnet/announcing-net-6/#microsoft-extensions-dependencyinjection-createasyncscope-apis + await using var scope = sf.CreateAsyncScope(); + + var hm = hmf(scope.ServiceProvider); + var handler = await hm.Start(ct).ConfigureAwait(false); + try + { + await handler(payload, ct).ConfigureAwait(false); + await hm.Stop(null, ct).ConfigureAwait(false); + } + catch (OperationCanceledException e) when (e.CancellationToken == ct) + { + throw; + } + catch (Exception e) + { + await hm.Stop(e, ct).ConfigureAwait(false); + } + } + + public ValueTask> Start(CancellationToken ct) + { + Handler handler = Handle; + return ValueTask.FromResult(handler); + } + + public ValueTask Stop(Exception? error, CancellationToken ct) => ValueTask.CompletedTask; +} diff --git a/src/LocalPost/Primitives.cs b/src/LocalPost/Primitives.cs new file mode 100644 index 0000000..5ecd97f --- /dev/null +++ b/src/LocalPost/Primitives.cs @@ -0,0 +1,27 @@ +namespace LocalPost; + +// int, 1 <= value <= int.MaxValue +internal readonly record struct PositiveInt +{ + public static implicit operator int(PositiveInt num) => num.Value; + + public static implicit operator PositiveInt(int num) => new(num); + public static implicit operator PositiveInt(short num) => new(num); + public static implicit operator PositiveInt(ushort num) => new(num); + + private readonly int _value; + public int Value => _value == 0 ? 1 : _value; // Default value + + private PositiveInt(int num) + { + if (num < 1) + throw new ArgumentOutOfRangeException(nameof(num), num, "Must be greater than or equal to 1"); + + _value = num; + } + + public void Deconstruct(out int value) + { + value = Value; + } +} diff --git a/src/LocalPost/QueueOptions.cs b/src/LocalPost/QueueOptions.cs deleted file mode 100644 index 23a2cb3..0000000 --- a/src/LocalPost/QueueOptions.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System.ComponentModel.DataAnnotations; - -namespace LocalPost; - -/// -/// Background queue configuration. -/// -public sealed record QueueOptions -{ - /// - /// How many messages to process in parallel. - /// - [Required] public ushort MaxConcurrency { get; set; } = ushort.MaxValue; -} diff --git a/src/LocalPost/RecordsSupport.cs b/src/LocalPost/RecordsSupport.cs deleted file mode 100644 index 0bc7e05..0000000 --- a/src/LocalPost/RecordsSupport.cs +++ /dev/null @@ -1,10 +0,0 @@ -// ReSharper disable once CheckNamespace -namespace System.Runtime.CompilerServices; - -using ComponentModel; - -// See https://bit.ly/3xSzC0Q -[EditorBrowsable(EditorBrowsableState.Never)] -internal static class IsExternalInit -{ -} diff --git a/src/LocalPost/Reflection.cs b/src/LocalPost/Reflection.cs index fb418a0..4f48d08 100644 --- a/src/LocalPost/Reflection.cs +++ b/src/LocalPost/Reflection.cs @@ -1,12 +1,16 @@ -using System.Diagnostics.CodeAnalysis; - -namespace LocalPost; +namespace LocalPost; [ExcludeFromCodeCoverage] internal static class Reflection { + // public static string FriendlyNameOf(string name) where T : INamedService => + public static string FriendlyNameOf(string? name) => FriendlyNameOf(typeof(T), name); + public static string FriendlyNameOf() => FriendlyNameOf(typeof(T)); + public static string FriendlyNameOf(Type type, string? name) => + FriendlyNameOf(type) + (string.IsNullOrEmpty(name) ? "" : $" (\"{name}\")"); + public static string FriendlyNameOf(Type type) => type.IsGenericType switch { true => type.Name.Split('`')[0] diff --git a/src/LocalPost/Resilience/HandlerStackEx.cs b/src/LocalPost/Resilience/HandlerStackEx.cs new file mode 100644 index 0000000..101a8db --- /dev/null +++ b/src/LocalPost/Resilience/HandlerStackEx.cs @@ -0,0 +1,20 @@ +using Polly; + +namespace LocalPost.Resilience; + +[PublicAPI] +public static class HandlerStackEx +{ + public static HandlerManagerFactory UsePollyPipeline(this HandlerManagerFactory hmf, + ResiliencePipeline pipeline) => hmf.TouchHandler(next => (context, ct) => + pipeline.ExecuteAsync(execCt => next(context, execCt), ct)); + + public static HandlerManagerFactory UsePollyPipeline(this HandlerManagerFactory hmf, + Action configure) + { + var builder = new ResiliencePipelineBuilder(); + configure(builder); + + return hmf.UsePollyPipeline(builder.Build()); + } +} diff --git a/src/LocalPost/globalusings.cs b/src/LocalPost/globalusings.cs new file mode 100644 index 0000000..2f865c2 --- /dev/null +++ b/src/LocalPost/globalusings.cs @@ -0,0 +1,3 @@ +global using JetBrains.Annotations; +global using System.Diagnostics.CodeAnalysis; +global using Microsoft.Extensions.Logging; diff --git a/tests/Directory.Build.props b/tests/Directory.Build.props new file mode 100644 index 0000000..5852105 --- /dev/null +++ b/tests/Directory.Build.props @@ -0,0 +1,25 @@ + + + + 13 + enable + enable + true + + false + + + + + + + + + + + + + + + + diff --git a/tests/LocalPost.KafkaConsumer.Tests/ConsumerTests.cs b/tests/LocalPost.KafkaConsumer.Tests/ConsumerTests.cs new file mode 100644 index 0000000..e192cc0 --- /dev/null +++ b/tests/LocalPost.KafkaConsumer.Tests/ConsumerTests.cs @@ -0,0 +1,86 @@ +using System.Text; +using Confluent.Kafka; +using LocalPost.KafkaConsumer.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace LocalPost.KafkaConsumer.Tests; + +public class ConsumerTests(ITestOutputHelper output) : IAsyncLifetime +{ + // Called for each test, since each test instantiates a new class instance + private readonly RedpandaContainer _container = new RedpandaBuilder() + .Build(); + + private const string Topic = "weather-forecasts"; + + public async Task InitializeAsync() + { + await _container.StartAsync(); + + using var producer = new ProducerBuilder(new ProducerConfig + { + BootstrapServers = _container.GetBootstrapAddress() + }).Build(); + + // Redpanda creates a topic automatically if it doesn't exist + await producer.ProduceAsync(Topic, new Message + { + Key = "London", + Value = "It will be rainy in London tomorrow" + }); + await producer.ProduceAsync(Topic, new Message + { + Key = "Paris", + Value = "It will be sunny in Paris tomorrow" + }); + } + + public async Task DisposeAsync() + { + await _container.StopAsync(); + } + + [Fact] + public async Task handles_messages() + { + var received = new List(); + + var hostBuilder = Host.CreateApplicationBuilder(); + hostBuilder.Services.AddKafkaConsumers(kafka => kafka + .AddConsumer("test-consumer", + HandlerStack.For(payload => received.Add(payload)) + .Scoped() + .DecodeString() + .UseKafkaPayload() + .Acknowledge() + .Trace() + ) + .Configure(co => + { + co.ClientConfig.BootstrapServers = _container.GetBootstrapAddress(); + // Already set, see above + // co.ClientConfig.GroupId = "test-consumer"; + co.Topics.Add(Topic); + co.ClientConfig.EnableAutoOffsetStore = false; // Manually acknowledge every message + // Otherwise the client attaches to the end of the topic, skipping all the published messages + co.ClientConfig.AutoOffsetReset = AutoOffsetReset.Earliest; + }) + .ValidateDataAnnotations()); + + var host = hostBuilder.Build(); + + try + { + await host.StartAsync(); + + await Task.Delay(1_000); // "App is working" + + received.Should().HaveCount(2); + } + finally + { + await host.StopAsync(); + } + } +} diff --git a/tests/LocalPost.KafkaConsumer.Tests/LocalPost.KafkaConsumer.Tests.csproj b/tests/LocalPost.KafkaConsumer.Tests/LocalPost.KafkaConsumer.Tests.csproj new file mode 100644 index 0000000..b4b8894 --- /dev/null +++ b/tests/LocalPost.KafkaConsumer.Tests/LocalPost.KafkaConsumer.Tests.csproj @@ -0,0 +1,19 @@ + + + + net6;net8 + + + + + + + + + + + + + + + diff --git a/tests/LocalPost.KafkaConsumer.Tests/RedpandaContainer.cs b/tests/LocalPost.KafkaConsumer.Tests/RedpandaContainer.cs new file mode 100644 index 0000000..435067e --- /dev/null +++ b/tests/LocalPost.KafkaConsumer.Tests/RedpandaContainer.cs @@ -0,0 +1,87 @@ +using System.Text; +using Docker.DotNet.Models; +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Configurations; +using DotNet.Testcontainers.Containers; + +namespace LocalPost.KafkaConsumer.Tests; + +// See also https://github.com/testcontainers/testcontainers-dotnet/blob/develop/src/Testcontainers.Kafka/KafkaBuilder.cs +public sealed class RedpandaBuilder : ContainerBuilder +{ + public const string RedpandaImage = "docker.redpanda.com/redpandadata/redpanda:v24.3.3"; + + public const ushort KafkaPort = 9092; + public const ushort KafkaAdminPort = 9644; + public const ushort PandaProxyPort = 8082; + public const ushort SchemaRegistryPort = 8081; + + public const string StartupScriptFilePath = "/testcontainers.sh"; + + public RedpandaBuilder() : this(new ContainerConfiguration()) + { + DockerResourceConfiguration = Init().DockerResourceConfiguration; + } + + private RedpandaBuilder(ContainerConfiguration resourceConfiguration) : base(resourceConfiguration) + { + DockerResourceConfiguration = resourceConfiguration; + } + + protected override RedpandaBuilder Clone(IResourceConfiguration resourceConfiguration) => + Merge(DockerResourceConfiguration, new ContainerConfiguration(resourceConfiguration)); + + protected override RedpandaBuilder Clone(IContainerConfiguration resourceConfiguration) => + Merge(DockerResourceConfiguration, new ContainerConfiguration(resourceConfiguration)); + + protected override RedpandaBuilder Merge(ContainerConfiguration oldValue, ContainerConfiguration newValue) => + new(new ContainerConfiguration(oldValue, newValue)); + + protected override ContainerConfiguration DockerResourceConfiguration { get; } + + public override RedpandaContainer Build() + { + Validate(); + return new RedpandaContainer(DockerResourceConfiguration); + } + + protected override RedpandaBuilder Init() + { + return base.Init() + .WithImage(RedpandaImage) + .WithPortBinding(KafkaPort, true) + .WithPortBinding(KafkaAdminPort, true) + .WithPortBinding(PandaProxyPort, true) + .WithPortBinding(SchemaRegistryPort, true) + .WithEntrypoint("/bin/sh", "-c") + .WithCommand("while [ ! -f " + StartupScriptFilePath + " ]; do sleep 0.1; done; " + StartupScriptFilePath) + .WithWaitStrategy(Wait.ForUnixContainer().UntilMessageIsLogged("Started Kafka API server")) + .WithStartupCallback((container, ct) => + { + string[] cmd = + [ + "rpk", "redpanda", "start", + "--smp 1", + "--mode dev-container", + $"--kafka-addr internal://0.0.0.0:29092,external://0.0.0.0:{KafkaPort}", + $"--advertise-kafka-addr internal://{container.IpAddress}:29092,external://{container.Hostname}:{container.GetMappedPublicPort(KafkaPort)}", + $"--pandaproxy-addr internal://0.0.0.0:28082,external://0.0.0.0:{PandaProxyPort}", + $"--advertise-pandaproxy-addr internal://{container.IpAddress}:28082,external://{container.Hostname}:{container.GetMappedPublicPort(PandaProxyPort)}", + $"--schema-registry-addr internal://0.0.0.0:28081,external://0.0.0.0:{SchemaRegistryPort}", + ]; + var startupScript = "#!/bin/sh" + '\n' + '\n' + string.Join(' ', cmd) + '\n'; + + return container.CopyAsync(Encoding.Default.GetBytes(startupScript), StartupScriptFilePath, Unix.FileMode755, ct); + }); + } +} + +public sealed class RedpandaContainer(IContainerConfiguration configuration) : DockerContainer(configuration) +{ + public string GetSchemaRegistryAddress() => + new UriBuilder(Uri.UriSchemeHttp, Hostname, GetMappedPublicPort(RedpandaBuilder.SchemaRegistryPort)).ToString(); + + public string GetBootstrapAddress() => + // new UriBuilder("PLAINTEXT", Hostname, GetMappedPublicPort(RpBuilder.KafkaPort)).ToString(); + $"{Hostname}:{GetMappedPublicPort(RedpandaBuilder.KafkaPort)}"; +} diff --git a/tests/LocalPost.KafkaConsumer.Tests/globalusings.cs b/tests/LocalPost.KafkaConsumer.Tests/globalusings.cs new file mode 100644 index 0000000..8ab2b64 --- /dev/null +++ b/tests/LocalPost.KafkaConsumer.Tests/globalusings.cs @@ -0,0 +1,4 @@ +global using System.Threading.Tasks; +global using Xunit; +global using Xunit.Abstractions; +global using FluentAssertions; diff --git a/tests/LocalPost.SnsPublisher.Tests/LocalPost.SnsPublisher.Tests.csproj b/tests/LocalPost.SnsPublisher.Tests/LocalPost.SnsPublisher.Tests.csproj deleted file mode 100644 index 3779dba..0000000 --- a/tests/LocalPost.SnsPublisher.Tests/LocalPost.SnsPublisher.Tests.csproj +++ /dev/null @@ -1,26 +0,0 @@ - - - - netcoreapp3.1;net6;net7 - enable - - false - - - - - - - - - - - - - - - - - - - diff --git a/tests/LocalPost.SnsPublisher.Tests/Usings.cs b/tests/LocalPost.SnsPublisher.Tests/Usings.cs deleted file mode 100644 index c802f44..0000000 --- a/tests/LocalPost.SnsPublisher.Tests/Usings.cs +++ /dev/null @@ -1 +0,0 @@ -global using Xunit; diff --git a/tests/LocalPost.SqsConsumer.Tests/ConsumerTests.cs b/tests/LocalPost.SqsConsumer.Tests/ConsumerTests.cs new file mode 100644 index 0000000..44e93fb --- /dev/null +++ b/tests/LocalPost.SqsConsumer.Tests/ConsumerTests.cs @@ -0,0 +1,119 @@ +using Amazon.Extensions.NETCore.Setup; +using Amazon.Runtime; +using Amazon.SQS; +using LocalPost.Flow; +using LocalPost.SqsConsumer.DependencyInjection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Testcontainers.LocalStack; + +namespace LocalPost.SqsConsumer.Tests; + +public class ConsumerTests(ITestOutputHelper output) : IAsyncLifetime +{ + // Called for each test, since each test instantiates a new class instance + private readonly LocalStackContainer _container = new LocalStackBuilder() + .WithImage("localstack/localstack:4") + .WithEnvironment("SERVICES", "sqs") + .Build(); + + private readonly AWSCredentials _credentials = new BasicAWSCredentials("test", "test"); + + private const string QueueName = "weather-forecasts"; + + private string? _queueUrl; + + private IAmazonSQS CreateClient() => new AmazonSQSClient(_credentials, + new AmazonSQSConfig { ServiceURL = _container.GetConnectionString() }); + + public async Task InitializeAsync() + { + await _container.StartAsync(); + + var sqs = CreateClient(); + var createResponse = await sqs.CreateQueueAsync(QueueName); + _queueUrl = createResponse.QueueUrl; + } + + public Task DisposeAsync() => _container.StopAsync(); + + [Fact] + public async Task handles_messages() + { + var hostBuilder = Host.CreateApplicationBuilder(); + + var received = new List(); + + hostBuilder.Services + .AddDefaultAWSOptions(new AWSOptions() + { + DefaultClientConfig = { ServiceURL = _container.GetConnectionString() }, + Credentials = _credentials, + }) + .AddAWSService() + .AddSqsConsumers(sqs => sqs.AddConsumer(QueueName, + HandlerStack.For(payload => received.Add(payload)) + .Scoped() + .UseSqsPayload() + .Trace() + .LogExceptions() + .Acknowledge() // Acknowledge in any case, because we caught any possible exceptions before + )); + + var host = hostBuilder.Build(); + + await host.StartAsync(); + + var sqs = CreateClient(); + await sqs.SendMessageAsync(_queueUrl, "It will rainy in London tomorrow"); + + await Task.Delay(1_000); // "App is working" + + received.Should().HaveCount(1); + received[0].Should().Be("It will rainy in London tomorrow"); + + await host.StopAsync(); + } + + [Fact] + public async Task handles_batches() + { + var hostBuilder = Host.CreateApplicationBuilder(); + + var received = new List>(); + + hostBuilder.Services + .AddDefaultAWSOptions(new AWSOptions() + { + DefaultClientConfig = { ServiceURL = _container.GetConnectionString() }, + Credentials = _credentials, + }) + .AddAWSService() + .AddSqsConsumers(sqs => sqs.AddConsumer(QueueName, + HandlerStack.For>(payload => received.Add(payload)) + .Scoped() + .UseSqsPayload() + .Trace() + .LogExceptions() + .Acknowledge() // Will acknowledge in any case, as we already caught all the exceptions before + .Batch(10, TimeSpan.FromSeconds(1)) + )); + + var host = hostBuilder.Build(); + + await host.StartAsync(); + + var sqs = CreateClient(); + await sqs.SendMessageAsync(_queueUrl, "It will be rainy in London tomorrow"); + await sqs.SendMessageAsync(_queueUrl, "It will be sunny in Paris tomorrow"); + + await Task.Delay(3_000); // "App is working" + + received.Should().HaveCount(1); + received[0].Should().BeEquivalentTo( + "It will be rainy in London tomorrow", + "It will be sunny in Paris tomorrow"); + + await host.StopAsync(); + } +} diff --git a/tests/LocalPost.SqsConsumer.Tests/LocalPost.SqsConsumer.Tests.csproj b/tests/LocalPost.SqsConsumer.Tests/LocalPost.SqsConsumer.Tests.csproj index adf1316..dca90b3 100644 --- a/tests/LocalPost.SqsConsumer.Tests/LocalPost.SqsConsumer.Tests.csproj +++ b/tests/LocalPost.SqsConsumer.Tests/LocalPost.SqsConsumer.Tests.csproj @@ -1,22 +1,15 @@ - netcoreapp3.1;net6;net7 - enable - - false + net6;net8 - - - - + + - - - - + + diff --git a/tests/LocalPost.SqsConsumer.Tests/Usings.cs b/tests/LocalPost.SqsConsumer.Tests/Usings.cs deleted file mode 100644 index c802f44..0000000 --- a/tests/LocalPost.SqsConsumer.Tests/Usings.cs +++ /dev/null @@ -1 +0,0 @@ -global using Xunit; diff --git a/tests/LocalPost.SqsConsumer.Tests/globalusings.cs b/tests/LocalPost.SqsConsumer.Tests/globalusings.cs new file mode 100644 index 0000000..8ab2b64 --- /dev/null +++ b/tests/LocalPost.SqsConsumer.Tests/globalusings.cs @@ -0,0 +1,4 @@ +global using System.Threading.Tasks; +global using Xunit; +global using Xunit.Abstractions; +global using FluentAssertions; diff --git a/tests/LocalPost.Tests/AsyncEnumerableMergerTests.cs b/tests/LocalPost.Tests/AsyncEnumerableMergerTests.cs deleted file mode 100644 index 53a0ad1..0000000 --- a/tests/LocalPost.Tests/AsyncEnumerableMergerTests.cs +++ /dev/null @@ -1,144 +0,0 @@ -using System.Threading.Channels; -using FluentAssertions; - -namespace LocalPost.Tests; - -public class AsyncEnumerableMergerTests -{ - [Fact] - internal async Task aggregates_multiple_channels() - { - var source1 = Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false - }); - var source2 = Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false - }); - var results = new AsyncEnumerableMerger(new[] - { - source1.Reader.ReadAllAsync(), source2.Reader.ReadAllAsync() - }); - - async Task Produce() - { - await source1.Writer.WriteAsync(1); - await source2.Writer.WriteAsync(1); - await source1.Writer.WriteAsync(1); - - await Task.Delay(TimeSpan.FromSeconds(1)); - - source1.Writer.Complete(); - await source2.Writer.WriteAsync(4); - - await Task.Delay(TimeSpan.FromSeconds(1)); - - source2.Writer.Complete(); - } - - async Task Consume() - { - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - - var expect = new Queue(); - expect.Enqueue(1); - expect.Enqueue(1); - expect.Enqueue(1); - expect.Enqueue(4); - - await foreach (var r in results.WithCancellation(cts.Token)) - r.Should().Be(expect.Dequeue()); - - expect.Should().BeEmpty(); - } - - await Task.WhenAll(Produce(), Consume()); - } - - [Fact] - internal async Task aggregates_multiple_channels_over_time() - { - var source1 = Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false - }); - var source2 = Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false - }); - var results = new AsyncEnumerableMerger(true); - - async Task Produce() - { - await source1.Writer.WriteAsync(1); - await source2.Writer.WriteAsync(2); - await source1.Writer.WriteAsync(3); - - await Task.Delay(TimeSpan.FromSeconds(1)); // Does not matter - - results.Add(source1.Reader.ReadAllAsync()); - - source1.Writer.Complete(); - - await source2.Writer.WriteAsync(4); - - results.Add(source2.Reader.ReadAllAsync()); - - await Task.Delay(TimeSpan.FromSeconds(1)); - - source2.Writer.Complete(); - } - - async Task Consume() - { - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - - var expect = new Queue(); - expect.Enqueue(1); - expect.Enqueue(3); - expect.Enqueue(2); - expect.Enqueue(4); - - try - { - await foreach (var r in results.WithCancellation(cts.Token)) - { -// r.Should().Be(expect.Dequeue()); - expect.Dequeue(); - } - } - catch (OperationCanceledException e) when (e.CancellationToken == cts.Token) - { - // Should happen - } - - cts.IsCancellationRequested.Should().BeTrue(); - expect.Should().BeEmpty(); - } - - await Task.WhenAll(Produce(), Consume()); - } - - [Fact] - internal async Task aggregates_multiple_channels_permanently() - { - var sut = new AsyncEnumerableMerger(true); - var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5)); - - try - { - await foreach (var r in sut.WithCancellation(cts.Token)) - { - } - } - catch (OperationCanceledException) - { - cts.IsCancellationRequested.Should().BeTrue(); - } - } -} diff --git a/tests/LocalPost.Tests/BatchingAsyncEnumerableTests.cs b/tests/LocalPost.Tests/BatchingAsyncEnumerableTests.cs deleted file mode 100644 index e4f017c..0000000 --- a/tests/LocalPost.Tests/BatchingAsyncEnumerableTests.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System.Threading.Channels; -using FluentAssertions; - -namespace LocalPost.Tests; - -public class BatchingAsyncEnumerableTests -{ - [Fact] - internal async Task batches() - { - var source = Channel.CreateUnbounded(new UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false - }); - var results = source.Reader.ReadAllAsync().Batch( - () => new BoundedBatchBuilder(10, TimeSpan.FromSeconds(2))); - - async Task Produce() - { - await source.Writer.WriteAsync(1); - await source.Writer.WriteAsync(2); - await source.Writer.WriteAsync(3); - - await Task.Delay(TimeSpan.FromSeconds(3)); - - await source.Writer.WriteAsync(4); - await source.Writer.WriteAsync(5); - - source.Writer.Complete(); - } - - async Task Consume() - { - var expect = new Queue(); - expect.Enqueue(new[] { 1, 2, 3 }); - expect.Enqueue(new[] { 4, 5 }); - await foreach (var batch in results) - { - batch.Should().ContainInOrder(expect.Dequeue()); - } - - expect.Should().BeEmpty(); - } - - await Task.WhenAll(Produce(), Consume()); - } -} diff --git a/tests/LocalPost.Tests/LocalPost.Tests.csproj b/tests/LocalPost.Tests/LocalPost.Tests.csproj index 140b60e..bbb9b08 100644 --- a/tests/LocalPost.Tests/LocalPost.Tests.csproj +++ b/tests/LocalPost.Tests/LocalPost.Tests.csproj @@ -1,24 +1,9 @@ - netcoreapp3.1;net6;net7 - enable - - false + net6;net8 - - - - - - - - - - - - diff --git a/tests/LocalPost.Tests/PrimitivesTests.cs b/tests/LocalPost.Tests/PrimitivesTests.cs new file mode 100644 index 0000000..d4eae08 --- /dev/null +++ b/tests/LocalPost.Tests/PrimitivesTests.cs @@ -0,0 +1,28 @@ +namespace LocalPost.Tests; + +public class PrimitivesTests +{ + [Fact] + public void MaxSize_implicit_conversion() + { + PositiveInt batchSize = default; + int value = batchSize; + value.Should().Be(1); + + batchSize = 1; + value = batchSize; + value.Should().Be(1); + + batchSize = 2; + value = batchSize; + value.Should().Be(2); + + batchSize = (short)3; + value = batchSize; + value.Should().Be(3); + + batchSize = (ushort)4; + value = batchSize; + value.Should().Be(4); + } +} diff --git a/tests/LocalPost.Tests/Usings.cs b/tests/LocalPost.Tests/Usings.cs deleted file mode 100644 index c802f44..0000000 --- a/tests/LocalPost.Tests/Usings.cs +++ /dev/null @@ -1 +0,0 @@ -global using Xunit; diff --git a/tests/LocalPost.Tests/globalusings.cs b/tests/LocalPost.Tests/globalusings.cs new file mode 100644 index 0000000..8ab2b64 --- /dev/null +++ b/tests/LocalPost.Tests/globalusings.cs @@ -0,0 +1,4 @@ +global using System.Threading.Tasks; +global using Xunit; +global using Xunit.Abstractions; +global using FluentAssertions;