From 0b445782a5dd24dc5b2c1cf53a50f24f656377a7 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 09:04:00 +0200 Subject: [PATCH 01/53] test(06-01): add failing tests for PartitionKey and InvalidPartitionKeyException - InvalidPartitionKeyExceptionTests: 5 cases covering all 4 constructors and ArgumentException inheritance - PartitionKeyTests: 11 cases covering valid/invalid inputs, implicit conversions, equality --- .../InvalidPartitionKeyExceptionTests.cs | 72 +++++++++ .../Models/ValueObjects/PartitionKeyTests.cs | 144 ++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 tests/Atomizer.Tests/Exceptions/InvalidPartitionKeyExceptionTests.cs create mode 100644 tests/Atomizer.Tests/Models/ValueObjects/PartitionKeyTests.cs diff --git a/tests/Atomizer.Tests/Exceptions/InvalidPartitionKeyExceptionTests.cs b/tests/Atomizer.Tests/Exceptions/InvalidPartitionKeyExceptionTests.cs new file mode 100644 index 0000000..16d1761 --- /dev/null +++ b/tests/Atomizer.Tests/Exceptions/InvalidPartitionKeyExceptionTests.cs @@ -0,0 +1,72 @@ +using Atomizer.Exceptions; + +namespace Atomizer.Tests.Exceptions; + +/// +/// Unit tests for . +/// +public class InvalidPartitionKeyExceptionTests +{ + [Fact] + public void Constructor_WithMessage_ShouldSetMessage() + { + // Arrange & Act + var ex = new InvalidPartitionKeyException("test message"); + + // Assert + ex.Message.Should().Contain("test message"); + ex.InnerException.Should().BeNull(); + ex.ParamName.Should().BeNull(); + } + + [Fact] + public void Constructor_WithMessageAndInnerException_ShouldSetBoth() + { + // Arrange + var inner = new Exception("inner"); + + // Act + var ex = new InvalidPartitionKeyException("test message", inner); + + // Assert + ex.Message.Should().Contain("test message"); + ex.InnerException.Should().BeSameAs(inner); + } + + [Fact] + public void Constructor_WithMessageAndParamName_ShouldSetBoth() + { + // Arrange & Act + var ex = new InvalidPartitionKeyException("test message", "myParam"); + + // Assert + ex.Message.Should().Contain("test message"); + ex.ParamName.Should().Be("myParam"); + ex.InnerException.Should().BeNull(); + } + + [Fact] + public void Constructor_WithMessageParamNameAndInnerException_ShouldSetAll() + { + // Arrange + var inner = new Exception("inner"); + + // Act + var ex = new InvalidPartitionKeyException("test message", "myParam", inner); + + // Assert + ex.Message.Should().Contain("test message"); + ex.ParamName.Should().Be("myParam"); + ex.InnerException.Should().BeSameAs(inner); + } + + [Fact] + public void ShouldDerive_FromArgumentException() + { + // Arrange & Act + var ex = new InvalidPartitionKeyException("test"); + + // Assert + ex.Should().BeAssignableTo(); + } +} diff --git a/tests/Atomizer.Tests/Models/ValueObjects/PartitionKeyTests.cs b/tests/Atomizer.Tests/Models/ValueObjects/PartitionKeyTests.cs new file mode 100644 index 0000000..d322c7f --- /dev/null +++ b/tests/Atomizer.Tests/Models/ValueObjects/PartitionKeyTests.cs @@ -0,0 +1,144 @@ +using Atomizer.Exceptions; + +namespace Atomizer.Tests.Models.ValueObjects; + +/// +/// Unit tests for . +/// +public class PartitionKeyTests +{ + [Fact] + public void Constructor_WithValidKey_ShouldSucceed() + { + // Arrange & Act + var pk = new PartitionKey("orders"); + + // Assert + pk.Key.Should().Be("orders"); + } + + [Fact] + public void Constructor_WithEmptyString_ShouldThrowInvalidPartitionKeyException() + { + // Arrange & Act + Action act = () => new PartitionKey(""); + + // Assert + act.Should() + .Throw() + .And.ParamName.Should() + .Be("key"); + } + + [Fact] + public void Constructor_WithWhitespaceOnly_ShouldThrowInvalidPartitionKeyException() + { + // Arrange & Act + Action act = () => new PartitionKey(" "); + + // Assert + act.Should() + .Throw() + .And.ParamName.Should() + .Be("key"); + } + + [Fact] + public void Constructor_WithNull_ShouldThrowInvalidPartitionKeyException() + { + // Arrange & Act + Action act = () => new PartitionKey(null!); + + // Assert + act.Should() + .Throw() + .And.ParamName.Should() + .Be("key"); + } + + [Fact] + public void Constructor_WithKeyExceeding255Chars_ShouldThrowInvalidPartitionKeyException() + { + // Arrange + var longKey = new string('x', 256); + + // Act + Action act = () => new PartitionKey(longKey); + + // Assert + act.Should() + .Throw() + .And.ParamName.Should() + .Be("key"); + } + + [Fact] + public void Constructor_WithExactly255Chars_ShouldSucceed() + { + // Arrange + var key = new string('x', 255); + + // Act + var pk = new PartitionKey(key); + + // Assert + pk.Key.Should().Be(key); + } + + [Fact] + public void ImplicitConversionFromString_ShouldCreatePartitionKey() + { + // Arrange & Act + PartitionKey pk = "orders"; + + // Assert + pk.Key.Should().Be("orders"); + } + + [Fact] + public void ImplicitConversionToString_ShouldReturnKeyString() + { + // Arrange + var pk = new PartitionKey("orders"); + + // Act + string value = pk; + + // Assert + value.Should().Be("orders"); + } + + [Fact] + public void ToString_ShouldReturnKeyString() + { + // Arrange + var pk = new PartitionKey("orders"); + + // Act & Assert + pk.ToString().Should().Be("orders"); + } + + [Fact] + public void Equality_WithSameKey_ShouldBeEqual() + { + // Arrange + var pk1 = new PartitionKey("orders"); + var pk2 = new PartitionKey("orders"); + + // Assert + pk1.Should().Be(pk2); + (pk1 == pk2).Should().BeTrue(); + } + + [Fact] + public void Equality_WithDifferentKey_ShouldNotBeEqual() + { + // Arrange + var pk1 = new PartitionKey("orders"); + var pk2 = new PartitionKey("payments"); + + // Assert + pk1.Should().NotBe(pk2); + (pk1 != pk2).Should().BeTrue(); + } +} From 01bc89909904bdd1bd6780b65093325fd13118fb Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 09:05:27 +0200 Subject: [PATCH 02/53] feat(06-01): add PartitionKey value object and InvalidPartitionKeyException - InvalidPartitionKeyException derives from ArgumentException with all 4 constructors and XML docs - PartitionKey sealed value object: 255-char cap, IsNullOrWhiteSpace guard, implicit string conversions, ValueObject equality - Both files follow exact structural analog of JobKey / InvalidJobKeyException --- .../InvalidPartitionKeyException.cs | 39 ++++++++++++ .../Models/ValueObjects/PartitionKey.cs | 62 +++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 src/Atomizer/Exceptions/InvalidPartitionKeyException.cs create mode 100644 src/Atomizer/Models/ValueObjects/PartitionKey.cs diff --git a/src/Atomizer/Exceptions/InvalidPartitionKeyException.cs b/src/Atomizer/Exceptions/InvalidPartitionKeyException.cs new file mode 100644 index 0000000..dad518c --- /dev/null +++ b/src/Atomizer/Exceptions/InvalidPartitionKeyException.cs @@ -0,0 +1,39 @@ +namespace Atomizer.Exceptions; + +/// +/// Thrown when a is constructed with an invalid value. +/// +public class InvalidPartitionKeyException : ArgumentException +{ + /// + /// Initializes a new instance with the specified message. + /// + /// The error message. + public InvalidPartitionKeyException(string message) + : base(message) { } + + /// + /// Initializes a new instance with the specified message and inner exception. + /// + /// The error message. + /// The exception that caused this exception. + public InvalidPartitionKeyException(string message, Exception innerException) + : base(message, innerException) { } + + /// + /// Initializes a new instance with the specified message and parameter name. + /// + /// The error message. + /// The name of the parameter that caused the exception. + public InvalidPartitionKeyException(string message, string paramName) + : base(message, paramName) { } + + /// + /// Initializes a new instance with the specified message, parameter name, and inner exception. + /// + /// The error message. + /// The name of the parameter that caused the exception. + /// The exception that caused this exception. + public InvalidPartitionKeyException(string message, string paramName, Exception innerException) + : base(message, paramName, innerException) { } +} diff --git a/src/Atomizer/Models/ValueObjects/PartitionKey.cs b/src/Atomizer/Models/ValueObjects/PartitionKey.cs new file mode 100644 index 0000000..eb40760 --- /dev/null +++ b/src/Atomizer/Models/ValueObjects/PartitionKey.cs @@ -0,0 +1,62 @@ +using Atomizer.Exceptions; +using Atomizer.Models.Base; + +namespace Atomizer; + +/// +/// Identifies a FIFO partition for ordered job processing. Maximum length is 255 characters. +/// +public sealed class PartitionKey : ValueObject +{ + /// + /// Initializes a new with the specified key. + /// + /// The partition key. Must be non-empty and at most 255 characters. + public PartitionKey(string key) + { + if (string.IsNullOrWhiteSpace(key)) + { + throw new InvalidPartitionKeyException("Partition key cannot be null or empty.", nameof(key)); + } + + if (key.Length > 255) + { + throw new InvalidPartitionKeyException("Partition key cannot exceed 255 characters.", nameof(key)); + } + + Key = key; + } + + /// + /// Gets the partition key string. + /// + public string Key { get; } + + /// + /// Returns the partition key string. + /// + public override string ToString() => Key; + + /// + /// Implicitly converts a to its string representation. + /// + /// The partition key to convert. + /// The partition key string. + public static implicit operator string(PartitionKey partitionKey) => partitionKey.Key; + + /// + /// Implicitly converts a string to a . + /// + /// The partition key string to convert. + /// A new wrapping the string. + public static implicit operator PartitionKey(string key) => new PartitionKey(key); + + /// + /// Returns the partition key string as the sole equality component. + /// + /// An enumerable containing the partition key string. + protected override IEnumerable GetEqualityValues() + { + yield return Key; + } +} From 8df9ea1ecf0af1948e55eb443d8427859b516c6f Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 09:10:17 +0200 Subject: [PATCH 03/53] feat(06-02): add PartitionKey to EnqueueOptions and RecurringOptions - Add PartitionKey? PartitionKey to EnqueueOptions with XML docs - Add PartitionKey? PartitionKey to RecurringOptions with XML docs --- src/Atomizer/Abstractions/IAtomizerClient.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/Atomizer/Abstractions/IAtomizerClient.cs b/src/Atomizer/Abstractions/IAtomizerClient.cs index 0b7b658..974aa15 100644 --- a/src/Atomizer/Abstractions/IAtomizerClient.cs +++ b/src/Atomizer/Abstractions/IAtomizerClient.cs @@ -75,6 +75,15 @@ public sealed class EnqueueOptions /// Defaults to 3 attempts with 15 seconds delays /// public RetryStrategy RetryStrategy { get; set; } = RetryStrategy.Default; + + /// + /// The partition key used to enforce ordered (FIFO) processing within this queue. + /// + /// + /// When set, jobs sharing the same and queue are processed one at a time + /// in sequence-number order. Defaults to , meaning the job participates in no partition. + /// + public PartitionKey? PartitionKey { get; set; } } /// @@ -116,4 +125,14 @@ public sealed class RecurringOptions /// Defaults to true. /// public bool Enabled { get; set; } = true; + + /// + /// The partition key used to enforce ordered (FIFO) processing of recurring job occurrences. + /// + /// + /// When set, each occurrence enqueued from this schedule carries the same , + /// preventing overlapping occurrences of the same recurring job from processing concurrently. + /// Defaults to , meaning occurrences participate in no partition. + /// + public PartitionKey? PartitionKey { get; set; } } From ed38a9a3acc92e1a100d8c708a908272458044ca Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 09:10:52 +0200 Subject: [PATCH 04/53] feat(06-02): add PartitionKey, SequenceNumber, IsPartitionBlocked to AtomizerJob - Add PartitionKey? PartitionKey property with XML docs - Add long? SequenceNumber property with XML docs - Add IsPartitionBlocked computed property (blocks when Processing or Pending+retrying) - Extend Create() with PartitionKey? partitionKey = null parameter - Initialize PartitionKey and SequenceNumber = null in Create() object initializer --- src/Atomizer/Models/AtomizerJob.cs | 36 +++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/Atomizer/Models/AtomizerJob.cs b/src/Atomizer/Models/AtomizerJob.cs index 7945202..38c49bc 100644 --- a/src/Atomizer/Models/AtomizerJob.cs +++ b/src/Atomizer/Models/AtomizerJob.cs @@ -82,6 +82,36 @@ public class AtomizerJob : Model /// public string? IdempotencyKey { get; set; } + /// + /// Gets or sets the partition key that groups this job for ordered (FIFO) processing, + /// or if the job participates in no partition. + /// + public PartitionKey? PartitionKey { get; set; } + + /// + /// Gets or sets the monotonically increasing sequence number within the job's + /// (queue, partition key) group, or for unpartitioned jobs. + /// + /// + /// Assigned atomically by storage at insert time. A value of + /// indicates either an unpartitioned job or a job not yet inserted into storage. + /// + public long? SequenceNumber { get; set; } + + /// + /// Gets whether this job is currently holding its partition, preventing + /// later jobs in the same partition from being picked up. + /// + /// + /// A job holds its partition when it is actively , + /// or when it is with prior attempts (retrying). + /// Jobs without a always return . + /// + public bool IsPartitionBlocked => + PartitionKey != null && + (Status == AtomizerJobStatus.Processing || + (Status == AtomizerJobStatus.Pending && Attempts > 0)); + /// /// Gets or sets the list of error records from previous failed attempts. /// @@ -98,6 +128,7 @@ public class AtomizerJob : Model /// Optional retry strategy; defaults to . /// Optional key used to deduplicate identical jobs. /// Optional key linking this job to a recurring schedule. + /// Optional partition key for ordered (FIFO) processing within the queue. /// A new instance. public static AtomizerJob Create( QueueKey queueKey, @@ -107,7 +138,8 @@ public static AtomizerJob Create( DateTimeOffset scheduledAt, RetryStrategy? retryStrategy = null, string? idempotencyKey = null, - JobKey? scheduleJobKey = null + JobKey? scheduleJobKey = null, + PartitionKey? partitionKey = null ) { return new AtomizerJob @@ -124,6 +156,8 @@ public static AtomizerJob Create( UpdatedAt = createdAt, IdempotencyKey = idempotencyKey, ScheduleJobKey = scheduleJobKey, + PartitionKey = partitionKey, + SequenceNumber = null, }; } From de3dd05994400de10f7b9a48b854becb924393c8 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 09:11:23 +0200 Subject: [PATCH 05/53] feat(06-02): add PartitionKey to AtomizerSchedule - Add PartitionKey? PartitionKey property with XML docs - Extend Create() with PartitionKey? partitionKey = null parameter - Initialize PartitionKey = partitionKey in Create() object initializer --- src/Atomizer/Models/AtomizerSchedule.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Atomizer/Models/AtomizerSchedule.cs b/src/Atomizer/Models/AtomizerSchedule.cs index 706bcc8..1078499 100644 --- a/src/Atomizer/Models/AtomizerSchedule.cs +++ b/src/Atomizer/Models/AtomizerSchedule.cs @@ -58,6 +58,16 @@ public class AtomizerSchedule : Model /// public RetryStrategy RetryStrategy { get; set; } = RetryStrategy.Default; + /// + /// Gets or sets the partition key applied to each job occurrence enqueued from this schedule, + /// or if occurrences participate in no partition. + /// + /// + /// When set, the is forwarded to every created + /// by ScheduleProcessor, enabling FIFO ordering across recurring job occurrences. + /// + public PartitionKey? PartitionKey { get; set; } + /// /// Gets or sets the UTC time of the next scheduled occurrence. /// @@ -94,6 +104,7 @@ public class AtomizerSchedule : Model /// Maximum missed runs to catch up. Defaults to 5. /// Whether the schedule is active. Defaults to true. /// Optional retry strategy; defaults to . + /// Optional partition key forwarded to each job occurrence for FIFO ordering. /// A new instance. public static AtomizerSchedule Create( JobKey jobKey, @@ -106,7 +117,8 @@ public static AtomizerSchedule Create( MisfirePolicy misfirePolicy = MisfirePolicy.ExecuteNow, int maxCatchUp = 5, bool enabled = true, - RetryStrategy? retryStrategy = null + RetryStrategy? retryStrategy = null, + PartitionKey? partitionKey = null ) { var atomizerSchedule = new AtomizerSchedule @@ -124,6 +136,7 @@ public static AtomizerSchedule Create( RetryStrategy = retryStrategy ?? RetryStrategy.Default, CreatedAt = createdAt, UpdatedAt = createdAt, + PartitionKey = partitionKey, }; atomizerSchedule.NextRunAt = From 1cfe5055d40e50f3d9de04406c04f2de0c01cc67 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 09:17:20 +0200 Subject: [PATCH 06/53] feat(06-03): wire PartitionKey through AtomizerClient - Pass partitionKey: options.PartitionKey (named arg) to AtomizerJob.Create() in EnqueueInternalAsync - Pass options.PartitionKey as last positional arg to AtomizerSchedule.Create() in ScheduleRecurringAsync --- src/Atomizer/Core/AtomizerClient.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Atomizer/Core/AtomizerClient.cs b/src/Atomizer/Core/AtomizerClient.cs index 05ee77b..a6f3162 100644 --- a/src/Atomizer/Core/AtomizerClient.cs +++ b/src/Atomizer/Core/AtomizerClient.cs @@ -84,7 +84,8 @@ public async Task ScheduleRecurringAsync( options.MisfirePolicy, options.MaxCatchUp, options.Enabled, - options.RetryStrategy + options.RetryStrategy, + options.PartitionKey ); using var scope = _serviceScopeFactory.CreateScope(); @@ -107,7 +108,8 @@ CancellationToken ct _clock.UtcNow, when, options.RetryStrategy, - options.IdempotencyKey + options.IdempotencyKey, + partitionKey: options.PartitionKey ); using var scope = _serviceScopeFactory.CreateScope(); From 6d848dbefc737dd1b28be4c56e08537771c7caf5 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 09:18:40 +0200 Subject: [PATCH 07/53] test(06-03): add PartitionKey and IsPartitionBlocked unit tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 11 PartitionKeyTests already provided by Plan 01 (PartitionKeyTests.cs) - 4 AtomizerJobPartitionTests: null partition → false, Pending+0 → false, Processing → true, Pending+Attempts>0 → true - All 15 targeted tests pass; full suite 93 passed on net8.0 and net10.0 --- .../Models/AtomizerJobPartitionTests.cs | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/Atomizer.Tests/Models/AtomizerJobPartitionTests.cs diff --git a/tests/Atomizer.Tests/Models/AtomizerJobPartitionTests.cs b/tests/Atomizer.Tests/Models/AtomizerJobPartitionTests.cs new file mode 100644 index 0000000..db8a8fb --- /dev/null +++ b/tests/Atomizer.Tests/Models/AtomizerJobPartitionTests.cs @@ -0,0 +1,74 @@ +namespace Atomizer.Tests.Models; + +/// +/// Unit tests for . +/// +public class AtomizerJobPartitionTests +{ + private static AtomizerJob CreateJob(PartitionKey? partitionKey = null) + { + return AtomizerJob.Create( + QueueKey.Default, + typeof(object), + "{}", + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow, + partitionKey: partitionKey + ); + } + + [Fact] + public void IsPartitionBlocked_WhenPartitionKeyIsNull_ShouldReturnFalse() + { + // Arrange + var job = CreateJob(partitionKey: null); + + // Assert + job.IsPartitionBlocked.Should().BeFalse(); + } + + [Fact] + public void IsPartitionBlocked_WhenStatusIsPendingAndAttemptsIsZero_ShouldReturnFalse() + { + // Arrange + var job = CreateJob(partitionKey: "orders"); + // Create() already sets Status=Pending, Attempts=0 + + // Assert + job.IsPartitionBlocked.Should().BeFalse(); + } + + [Fact] + public void IsPartitionBlocked_WhenStatusIsProcessing_ShouldReturnTrue() + { + // Arrange + var job = CreateJob(partitionKey: "orders"); + job.Lease( + new LeaseToken($"worker:*:default:*:{Guid.NewGuid()}"), + DateTimeOffset.UtcNow, + TimeSpan.FromMinutes(10) + ); + // Status is now Processing + + // Assert + job.IsPartitionBlocked.Should().BeTrue(); + } + + [Fact] + public void IsPartitionBlocked_WhenStatusIsPendingAndAttemptsGreaterThanZero_ShouldReturnTrue() + { + // Arrange + var job = CreateJob(partitionKey: "orders"); + job.Lease( + new LeaseToken($"worker:*:default:*:{Guid.NewGuid()}"), + DateTimeOffset.UtcNow, + TimeSpan.FromMinutes(10) + ); + job.Attempt(); + job.Reschedule(DateTimeOffset.UtcNow.AddSeconds(15), DateTimeOffset.UtcNow); + // Status is now Pending, Attempts == 1 + + // Assert + job.IsPartitionBlocked.Should().BeTrue(); + } +} From c33d0ec8fc634131932ee82751a48f5145a550d9 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 09:50:32 +0200 Subject: [PATCH 08/53] fix(06): release partition on job cancellation in JobProcessor Add job.Release and UpdateJobsAsync in the OperationCanceledException catch branch so partitioned jobs unblock their partition immediately instead of waiting for the full visibility timeout to expire. Co-Authored-By: Claude Sonnet 4.6 --- src/Atomizer/Processing/JobProcessor.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Atomizer/Processing/JobProcessor.cs b/src/Atomizer/Processing/JobProcessor.cs index 4315469..b6b6b7f 100644 --- a/src/Atomizer/Processing/JobProcessor.cs +++ b/src/Atomizer/Processing/JobProcessor.cs @@ -63,6 +63,11 @@ public async Task ProcessAsync(AtomizerJob job, CancellationToken ct) catch (OperationCanceledException) when (ct.IsCancellationRequested) { _logger.LogWarning("Operation cancelled while processing job {JobId} on '{Queue}'", job.Id, job.QueueKey); + + // Release the job so its partition (if any) is not blocked for the full visibility timeout. + job.Release(_clock.UtcNow); + using var scope = _serviceScopeFactory.CreateScope(); + await scope.Storage.UpdateJobsAsync(new[] { job }, CancellationToken.None); } catch (Exception ex) { From e3117780307d8be6f1fd6946e639ac4c80e86d59 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 09:50:47 +0200 Subject: [PATCH 09/53] fix(06): seal InvalidPartitionKeyException and AtomizerClient Apply the mandatory project convention that all public implementation classes are sealed. Both types were introduced in this phase without the sealed modifier. Co-Authored-By: Claude Sonnet 4.6 --- src/Atomizer/Core/AtomizerClient.cs | 2 +- src/Atomizer/Exceptions/InvalidPartitionKeyException.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Atomizer/Core/AtomizerClient.cs b/src/Atomizer/Core/AtomizerClient.cs index a6f3162..409898c 100644 --- a/src/Atomizer/Core/AtomizerClient.cs +++ b/src/Atomizer/Core/AtomizerClient.cs @@ -7,7 +7,7 @@ namespace Atomizer.Core; /// Default implementation of that serializes payloads /// and delegates to the configured . /// -public class AtomizerClient : IAtomizerClient +public sealed class AtomizerClient : IAtomizerClient { private readonly IAtomizerServiceScopeFactory _serviceScopeFactory; private readonly IAtomizerJobSerializer _jobSerializer; diff --git a/src/Atomizer/Exceptions/InvalidPartitionKeyException.cs b/src/Atomizer/Exceptions/InvalidPartitionKeyException.cs index dad518c..59d4495 100644 --- a/src/Atomizer/Exceptions/InvalidPartitionKeyException.cs +++ b/src/Atomizer/Exceptions/InvalidPartitionKeyException.cs @@ -3,7 +3,7 @@ namespace Atomizer.Exceptions; /// /// Thrown when a is constructed with an invalid value. /// -public class InvalidPartitionKeyException : ArgumentException +public sealed class InvalidPartitionKeyException : ArgumentException { /// /// Initializes a new instance with the specified message. From 86b4d9c6dd7e5e0e4ac45a36203d3095deb77025 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 09:51:05 +0200 Subject: [PATCH 10/53] fix(06): guard MarkAsCompleted/MarkAsFailed against invalid status Add Processing status pre-condition checks matching the guards already present on Lease, Release, and Attempt, preventing double-complete, double-fail, and completing/failing a job that was never leased. Co-Authored-By: Claude Sonnet 4.6 --- src/Atomizer/Models/AtomizerJob.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Atomizer/Models/AtomizerJob.cs b/src/Atomizer/Models/AtomizerJob.cs index 38c49bc..028708d 100644 --- a/src/Atomizer/Models/AtomizerJob.cs +++ b/src/Atomizer/Models/AtomizerJob.cs @@ -216,6 +216,11 @@ public void Attempt() /// The UTC time the job completed. public void MarkAsCompleted(DateTimeOffset completedAt) { + if (Status != AtomizerJobStatus.Processing) + { + throw new InvalidOperationException("Job must be in Processing status to mark as completed."); + } + CompletedAt = completedAt; UpdatedAt = completedAt; Status = AtomizerJobStatus.Completed; @@ -229,6 +234,11 @@ public void MarkAsCompleted(DateTimeOffset completedAt) /// The UTC time the job was permanently failed. public void MarkAsFailed(DateTimeOffset failedAt) { + if (Status != AtomizerJobStatus.Processing) + { + throw new InvalidOperationException("Job must be in Processing status to mark as failed."); + } + FailedAt = failedAt; UpdatedAt = failedAt; Status = AtomizerJobStatus.Failed; From bc18b0a8989177b4e1db70f521c1340d25f52a2a Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 09:51:16 +0200 Subject: [PATCH 11/53] fix(06): remove null-forgiving operator on PayloadType.FullName Replace job.PayloadType!.FullName with job.PayloadType?.FullName to avoid a potential NullReferenceException if a job is ever constructed without a PayloadType (e.g. from storage deserialization). Co-Authored-By: Claude Sonnet 4.6 --- src/Atomizer/Core/AtomizerClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Atomizer/Core/AtomizerClient.cs b/src/Atomizer/Core/AtomizerClient.cs index 409898c..5044dac 100644 --- a/src/Atomizer/Core/AtomizerClient.cs +++ b/src/Atomizer/Core/AtomizerClient.cs @@ -118,7 +118,7 @@ CancellationToken ct _logger.LogDebug( "Enqueuing job {JobId} with payload type {PayloadType} to queue {QueueKey} at {ScheduledAt}", jobId, - job.PayloadType!.FullName, + job.PayloadType?.FullName, job.QueueKey, job.ScheduledAt ); From 2e4d20499f2e351449a592aa09c103a06f579cd0 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 09:51:47 +0200 Subject: [PATCH 12/53] fix(06): make string-to-PartitionKey conversion explicit Change the implicit string operator to explicit so callers cannot accidentally trigger InvalidPartitionKeyException through a bare assignment. Update the corresponding test to use an explicit cast and rename the test method accordingly. Co-Authored-By: Claude Sonnet 4.6 --- src/Atomizer/Models/ValueObjects/PartitionKey.cs | 4 ++-- tests/Atomizer.Tests/Models/AtomizerJobPartitionTests.cs | 6 +++--- .../Atomizer.Tests/Models/ValueObjects/PartitionKeyTests.cs | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Atomizer/Models/ValueObjects/PartitionKey.cs b/src/Atomizer/Models/ValueObjects/PartitionKey.cs index eb40760..e1ddda6 100644 --- a/src/Atomizer/Models/ValueObjects/PartitionKey.cs +++ b/src/Atomizer/Models/ValueObjects/PartitionKey.cs @@ -45,11 +45,11 @@ public PartitionKey(string key) public static implicit operator string(PartitionKey partitionKey) => partitionKey.Key; /// - /// Implicitly converts a string to a . + /// Explicitly converts a string to a . /// /// The partition key string to convert. /// A new wrapping the string. - public static implicit operator PartitionKey(string key) => new PartitionKey(key); + public static explicit operator PartitionKey(string key) => new PartitionKey(key); /// /// Returns the partition key string as the sole equality component. diff --git a/tests/Atomizer.Tests/Models/AtomizerJobPartitionTests.cs b/tests/Atomizer.Tests/Models/AtomizerJobPartitionTests.cs index db8a8fb..eea10e3 100644 --- a/tests/Atomizer.Tests/Models/AtomizerJobPartitionTests.cs +++ b/tests/Atomizer.Tests/Models/AtomizerJobPartitionTests.cs @@ -31,7 +31,7 @@ public void IsPartitionBlocked_WhenPartitionKeyIsNull_ShouldReturnFalse() public void IsPartitionBlocked_WhenStatusIsPendingAndAttemptsIsZero_ShouldReturnFalse() { // Arrange - var job = CreateJob(partitionKey: "orders"); + var job = CreateJob(partitionKey: new PartitionKey("orders")); // Create() already sets Status=Pending, Attempts=0 // Assert @@ -42,7 +42,7 @@ public void IsPartitionBlocked_WhenStatusIsPendingAndAttemptsIsZero_ShouldReturn public void IsPartitionBlocked_WhenStatusIsProcessing_ShouldReturnTrue() { // Arrange - var job = CreateJob(partitionKey: "orders"); + var job = CreateJob(partitionKey: new PartitionKey("orders")); job.Lease( new LeaseToken($"worker:*:default:*:{Guid.NewGuid()}"), DateTimeOffset.UtcNow, @@ -58,7 +58,7 @@ public void IsPartitionBlocked_WhenStatusIsProcessing_ShouldReturnTrue() public void IsPartitionBlocked_WhenStatusIsPendingAndAttemptsGreaterThanZero_ShouldReturnTrue() { // Arrange - var job = CreateJob(partitionKey: "orders"); + var job = CreateJob(partitionKey: new PartitionKey("orders")); job.Lease( new LeaseToken($"worker:*:default:*:{Guid.NewGuid()}"), DateTimeOffset.UtcNow, diff --git a/tests/Atomizer.Tests/Models/ValueObjects/PartitionKeyTests.cs b/tests/Atomizer.Tests/Models/ValueObjects/PartitionKeyTests.cs index d322c7f..873cfd6 100644 --- a/tests/Atomizer.Tests/Models/ValueObjects/PartitionKeyTests.cs +++ b/tests/Atomizer.Tests/Models/ValueObjects/PartitionKeyTests.cs @@ -86,10 +86,10 @@ public void Constructor_WithExactly255Chars_ShouldSucceed() } [Fact] - public void ImplicitConversionFromString_ShouldCreatePartitionKey() + public void ExplicitConversionFromString_ShouldCreatePartitionKey() { // Arrange & Act - PartitionKey pk = "orders"; + var pk = (PartitionKey)"orders"; // Assert pk.Key.Should().Be("orders"); From a41f7f0aaa02522118b17c60736e6dd3520615d1 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 12:59:29 +0200 Subject: [PATCH 13/53] chore(07-01): add xunit.v3, AwesomeAssertions, NSubstitute to test utilities csproj - Remove duplicate enable - Add LangVersion 12 and IsTestProject true - Add xunit.v3.extensibility.core 2.0.1 (library project; not executable) - Add AwesomeAssertions 9.1.0 and NSubstitute 5.3.0 - Add global Xunit using for [Fact]/[Theory] in abstract base classes --- .../Atomizer.Tests.Utilities.csproj | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/Atomizer.Tests.Utilities/Atomizer.Tests.Utilities.csproj b/tests/Atomizer.Tests.Utilities/Atomizer.Tests.Utilities.csproj index c35f584..2cf2465 100644 --- a/tests/Atomizer.Tests.Utilities/Atomizer.Tests.Utilities.csproj +++ b/tests/Atomizer.Tests.Utilities/Atomizer.Tests.Utilities.csproj @@ -1,14 +1,21 @@ - + net6.0;net8.0;net10.0 enable false - enable true + 12 + true - + + + + + + + From dfd0666f9b3b13418950cc0e7810c3301daff714 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 13:00:08 +0200 Subject: [PATCH 14/53] docs(07-01): add FIFO contract remarks to IAtomizerStorage InsertAsync and GetDueJobsAsync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - InsertAsync: documents SequenceNumber mutation contract (D-04, D-05, D-06) — partitioned jobs get monotonically increasing sequence number scoped to (queue, partition key); unpartitioned jobs keep null; idempotency key collision assigns existing sequence number - GetDueJobsAsync: documents three-rule FIFO ordering contract (D-02) — at most one job per partition (lowest sequence number) — excludes partitions with Processing or retrying Pending jobs — unpartitioned jobs unaffected --- src/Atomizer/Abstractions/IAtomizerStorage.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/Atomizer/Abstractions/IAtomizerStorage.cs b/src/Atomizer/Abstractions/IAtomizerStorage.cs index 2f04e17..b4359ba 100644 --- a/src/Atomizer/Abstractions/IAtomizerStorage.cs +++ b/src/Atomizer/Abstractions/IAtomizerStorage.cs @@ -11,6 +11,13 @@ public interface IAtomizerStorage /// The Atomizer job to be inserted. /// Cancellation token to cancel the operation. /// The unique identifier of the inserted job. + /// + /// For partitioned jobs ( is not ), implementations + /// must assign a monotonically increasing scoped to the + /// (queue, partition key) before returning. For unpartitioned jobs, + /// must remain . On an idempotency key collision, the existing job's sequence number + /// is assigned to the passed-in job. + /// Task InsertAsync(AtomizerJob job, CancellationToken cancellationToken); /// @@ -29,6 +36,15 @@ public interface IAtomizerStorage /// The maximum number of jobs to retrieve in this batch. /// Cancellation token to cancel the operation. /// A list of due Atomizer jobs. + /// + /// When partition keys are in use, this method enforces FIFO ordering: + /// + /// At most one job per (queue, partition key) is returned — the job with the lowest sequence number. + /// A partition is excluded entirely if any job within it is + /// or with prior attempts (Attempts > 0). + /// Jobs without a partition key are unaffected and returned normally alongside partitioned jobs. + /// + /// Task> GetDueJobsAsync( QueueKey queueKey, DateTimeOffset now, From 4eea4996112a02d2b653b1b43920383879bc5448 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 13:03:50 +0200 Subject: [PATCH 15/53] feat(07-02): add AtomizerStorageContractTests abstract base class MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Seven [Fact] methods covering FIFO-07 (2), FIFO-08 (2), FIFO-09 (3) - IAsyncLifetime with ValueTask lifecycle (xUnit v3) - Protected abstract CreateStorage() factory for backend subclasses - No concrete subclass — Phases 8 and 9 provide those (D-11) --- .../AtomizerStorageContractTests.cs | 228 ++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs diff --git a/tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs b/tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs new file mode 100644 index 0000000..155cfdc --- /dev/null +++ b/tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs @@ -0,0 +1,228 @@ +using Atomizer.Abstractions; +using Atomizer.Core; +using Atomizer.Tests.Utilities.Stubs; +using Atomizer.Tests.Utilities.TestJobs; +using AwesomeAssertions; +using NSubstitute; + +namespace Atomizer.Tests.Utilities.StorageContract; + +/// +/// Abstract contract test base that verifies FIFO storage semantics (FIFO-07, FIFO-08, FIFO-09). +/// Subclass this in each storage backend test project and implement . +/// +public abstract class AtomizerStorageContractTests : IAsyncLifetime +{ + private readonly IAtomizerClock _clock = Substitute.For(); + protected readonly DateTimeOffset _now = DateTimeOffset.UtcNow; + protected IAtomizerStorage _sut = null!; + + /// + /// Creates a fresh storage instance for the test run. + /// + /// A new implementation to test. + protected abstract IAtomizerStorage CreateStorage(); + + /// + public ValueTask InitializeAsync() + { + _clock.UtcNow.Returns(_now); + _sut = CreateStorage(); + return ValueTask.CompletedTask; + } + + /// + public virtual ValueTask DisposeAsync() => ValueTask.CompletedTask; + + // ------------------------------------------------------------------ + // FIFO-09: SequenceNumber assignment on InsertAsync + // ------------------------------------------------------------------ + + /// + /// FIFO-09: Partitioned jobs receive monotonically increasing sequence numbers. + /// + [Fact] + public async Task InsertAsync_WhenPartitionedJob_ShouldAssignMonotonicallyIncreasingSequenceNumber() + { + // Arrange + var partitionKey = new PartitionKey("order-123"); + var job1 = CreateJob(partitionKey: partitionKey); + var job2 = CreateJob(partitionKey: partitionKey); + + // Act + await _sut.InsertAsync(job1, CancellationToken.None); + await _sut.InsertAsync(job2, CancellationToken.None); + + // Assert + job1.SequenceNumber.Should().NotBeNull(); + job2.SequenceNumber.Should().NotBeNull(); + job2.SequenceNumber.Should().BeGreaterThan(job1.SequenceNumber!.Value); + } + + /// + /// FIFO-09: Unpartitioned jobs do not receive a sequence number. + /// + [Fact] + public async Task InsertAsync_WhenUnpartitionedJob_ShouldNotAssignSequenceNumber() + { + // Arrange + var job = CreateJob(); + + // Act + await _sut.InsertAsync(job, CancellationToken.None); + + // Assert + job.SequenceNumber.Should().BeNull(); + } + + /// + /// FIFO-09 / D-06: On an idempotency key collision, the existing job's sequence number + /// is assigned to the passed-in job. + /// + [Fact] + public async Task InsertAsync_WhenIdempotencyKeyCollision_ShouldAssignExistingSequenceNumber() + { + // Arrange + var partitionKey = new PartitionKey("orders"); + const string idempotencyKey = "idem-key-1"; + + var job1 = CreateJob(partitionKey: partitionKey, idempotencyKey: idempotencyKey); + await _sut.InsertAsync(job1, CancellationToken.None); + + var job2 = CreateJob(partitionKey: partitionKey, idempotencyKey: idempotencyKey); + + // Act + await _sut.InsertAsync(job2, CancellationToken.None); + + // Assert + job1.SequenceNumber.Should().NotBeNull(); + job2.SequenceNumber.Should().Be(job1.SequenceNumber); + } + + // ------------------------------------------------------------------ + // FIFO-07: GetDueJobsAsync returns at most one job per partition + // ------------------------------------------------------------------ + + /// + /// FIFO-07: When multiple jobs share a partition key, only the lowest-sequence-number + /// job is returned by . + /// + [Fact] + public async Task GetDueJobsAsync_WhenMultipleJobsInSamePartition_ShouldReturnOnlyLowestSequenceNumber() + { + // Arrange + var partitionKey = new PartitionKey("batch-key"); + var job1 = CreateJob(partitionKey: partitionKey); + var job2 = CreateJob(partitionKey: partitionKey); + + await _sut.InsertAsync(job1, CancellationToken.None); + await _sut.InsertAsync(job2, CancellationToken.None); + + // Act + var result = await _sut.GetDueJobsAsync(QueueKey.Default, _now, batchSize: 10, CancellationToken.None); + + // Assert + result.Should().HaveCount(1); + result.Single().Id.Should().Be(job1.Id); + } + + /// + /// FIFO-07: Unpartitioned jobs are returned alongside the head of each partition. + /// + [Fact] + public async Task GetDueJobsAsync_WhenUnpartitionedJobsExist_ShouldReturnThemAlongsidePartitionedJobs() + { + // Arrange + var partitionedJob = CreateJob(partitionKey: new PartitionKey("p1")); + var unpartitionedJob = CreateJob(); + + await _sut.InsertAsync(partitionedJob, CancellationToken.None); + await _sut.InsertAsync(unpartitionedJob, CancellationToken.None); + + // Act + var result = await _sut.GetDueJobsAsync(QueueKey.Default, _now, batchSize: 10, CancellationToken.None); + + // Assert + result.Should().HaveCount(2); + result.Should().Contain(j => j.Id == partitionedJob.Id); + result.Should().Contain(j => j.Id == unpartitionedJob.Id); + } + + // ------------------------------------------------------------------ + // FIFO-08: GetDueJobsAsync excludes entire partition when head is blocked + // ------------------------------------------------------------------ + + /// + /// FIFO-08: When the head job of a partition is Processing, the entire partition is excluded. + /// + [Fact] + public async Task GetDueJobsAsync_WhenPartitionIsBlockedByProcessing_ShouldExcludeEntirePartition() + { + // Arrange + var partitionKey = new PartitionKey("blocked-p"); + var job1 = CreateJob(partitionKey: partitionKey); + var job2 = CreateJob(partitionKey: partitionKey); + + await _sut.InsertAsync(job1, CancellationToken.None); + await _sut.InsertAsync(job2, CancellationToken.None); + + // Transition job1 to Processing and persist + job1.Lease(FakeDataFactory.LeaseToken(), _now, TimeSpan.FromMinutes(10)); + await _sut.UpdateJobsAsync([job1], CancellationToken.None); + + // Act + var result = await _sut.GetDueJobsAsync(QueueKey.Default, _now, batchSize: 10, CancellationToken.None); + + // Assert — entire partition invisible while job1 is Processing + result.Should().BeEmpty(); + } + + /// + /// FIFO-08: When the head job of a partition is Pending with prior attempts (retrying), + /// the entire partition is excluded. + /// + [Fact] + public async Task GetDueJobsAsync_WhenPartitionIsBlockedByPendingWithAttempts_ShouldExcludeEntirePartition() + { + // Arrange + var partitionKey = new PartitionKey("retry-p"); + var job1 = CreateJob(partitionKey: partitionKey); + var job2 = CreateJob(partitionKey: partitionKey); + + await _sut.InsertAsync(job1, CancellationToken.None); + await _sut.InsertAsync(job2, CancellationToken.None); + + // Simulate retry state: Lease → Attempt → Reschedule (Pending with Attempts = 1) + job1.Lease(FakeDataFactory.LeaseToken(), _now, TimeSpan.FromMinutes(10)); + job1.Attempt(); + job1.Reschedule(_now, _now); + await _sut.UpdateJobsAsync([job1], CancellationToken.None); + + // Act + var result = await _sut.GetDueJobsAsync(QueueKey.Default, _now, batchSize: 10, CancellationToken.None); + + // Assert — partition blocked while job1 is Pending with Attempts > 0 + result.Should().BeEmpty(); + } + + // ------------------------------------------------------------------ + // Helper + // ------------------------------------------------------------------ + + private AtomizerJob CreateJob( + PartitionKey? partitionKey = null, + string? idempotencyKey = null, + QueueKey? queueKey = null + ) + { + return AtomizerJob.Create( + queueKey ?? QueueKey.Default, + typeof(WriteLineJob), + "{}", + _now, + _now, + idempotencyKey: idempotencyKey, + partitionKey: partitionKey + ); + } +} From b2858d2b8578b4b18ccd4379e89cbee3742e8a0d Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 13:11:34 +0200 Subject: [PATCH 16/53] fix(07): WR-02 fix copy-paste error in UpdateSchedulesAsync XML doc Co-Authored-By: Claude Sonnet 4.6 --- src/Atomizer/Abstractions/IAtomizerStorage.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Atomizer/Abstractions/IAtomizerStorage.cs b/src/Atomizer/Abstractions/IAtomizerStorage.cs index b4359ba..64b87ff 100644 --- a/src/Atomizer/Abstractions/IAtomizerStorage.cs +++ b/src/Atomizer/Abstractions/IAtomizerStorage.cs @@ -71,7 +71,7 @@ CancellationToken cancellationToken Task UpsertScheduleAsync(AtomizerSchedule schedule, CancellationToken cancellationToken); /// - /// Updates a range of existing schedules jobs in the storage. + /// Updates a range of existing Atomizer schedules in the storage. /// /// The collection of Atomizer schedules to be updated. /// Cancellation token to cancel the operation. From 7a91391616145ac14987e809df3d6775b5b76636 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 13:13:02 +0200 Subject: [PATCH 17/53] fix(07): CR-01 CR-02 WR-01 WR-03 WR-04 fix contract test base correctness and coverage - CR-01: thread IAtomizerClock into CreateStorage(clock) so storage uses the test-controlled clock instead of its own internal instance - CR-02: add pre-condition to class XML summary documenting that CreateStorage must return a FIFO-compliant implementation - WR-01: move _now initialisation from field-initializer into InitializeAsync so each test gets a fresh, clock-stub-aligned timestamp - WR-03: add cross-queue partition isolation test - WR-04: add ReleaseLeasedAsync partition-unblocking test Co-Authored-By: Claude Sonnet 4.6 --- .../AtomizerStorageContractTests.cs | 72 ++++++++++++++++++- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs b/tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs index 155cfdc..e85bd64 100644 --- a/tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs +++ b/tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs @@ -10,24 +10,32 @@ namespace Atomizer.Tests.Utilities.StorageContract; /// /// Abstract contract test base that verifies FIFO storage semantics (FIFO-07, FIFO-08, FIFO-09). /// Subclass this in each storage backend test project and implement . +/// +/// Pre-condition: The returned by +/// must fully implement the FIFO partition-blocking rules +/// described in . Tests will fail if the +/// implementation does not enforce these rules. +/// /// public abstract class AtomizerStorageContractTests : IAsyncLifetime { private readonly IAtomizerClock _clock = Substitute.For(); - protected readonly DateTimeOffset _now = DateTimeOffset.UtcNow; + protected DateTimeOffset _now; protected IAtomizerStorage _sut = null!; /// /// Creates a fresh storage instance for the test run. /// + /// The clock instance the storage implementation must use. /// A new implementation to test. - protected abstract IAtomizerStorage CreateStorage(); + protected abstract IAtomizerStorage CreateStorage(IAtomizerClock clock); /// public ValueTask InitializeAsync() { + _now = DateTimeOffset.UtcNow; _clock.UtcNow.Returns(_now); - _sut = CreateStorage(); + _sut = CreateStorage(_clock); return ValueTask.CompletedTask; } @@ -205,6 +213,64 @@ public async Task GetDueJobsAsync_WhenPartitionIsBlockedByPendingWithAttempts_Sh result.Should().BeEmpty(); } + /// + /// FIFO-08: Two jobs sharing the same partition key string but in different queues + /// are treated as independent partitions. + /// + [Fact] + public async Task GetDueJobsAsync_WhenSamePartitionKeyInDifferentQueues_ShouldTreatAsIndependent() + { + var partitionKey = new PartitionKey("shared-key"); + var queueA = QueueKey.Default; + var queueB = new QueueKey("secondary"); + + var jobA = CreateJob(partitionKey: partitionKey, queueKey: queueA); + var jobB = CreateJob(partitionKey: partitionKey, queueKey: queueB); + + await _sut.InsertAsync(jobA, CancellationToken.None); + await _sut.InsertAsync(jobB, CancellationToken.None); + + jobA.Lease(FakeDataFactory.LeaseToken(), _now, TimeSpan.FromMinutes(10)); + await _sut.UpdateJobsAsync([jobA], CancellationToken.None); + + var result = await _sut.GetDueJobsAsync(queueB, _now, batchSize: 10, CancellationToken.None); + + result.Should().HaveCount(1); + result.Single().Id.Should().Be(jobB.Id); + } + + // ------------------------------------------------------------------ + // ReleaseLeasedAsync: partition unblocking + // ------------------------------------------------------------------ + + /// + /// Releasing a leased partition head must clear VisibleAt and make the job visible again, + /// unblocking the entire partition. + /// + [Fact] + public async Task ReleaseLeasedAsync_WhenPartitionHeadReleased_ShouldUnblockPartition() + { + var partitionKey = new PartitionKey("release-p"); + var job1 = CreateJob(partitionKey: partitionKey); + var job2 = CreateJob(partitionKey: partitionKey); + + await _sut.InsertAsync(job1, CancellationToken.None); + await _sut.InsertAsync(job2, CancellationToken.None); + + var leaseToken = FakeDataFactory.LeaseToken(); + job1.Lease(leaseToken, _now, TimeSpan.FromMinutes(10)); + await _sut.UpdateJobsAsync([job1], CancellationToken.None); + + var blocked = await _sut.GetDueJobsAsync(QueueKey.Default, _now, batchSize: 10, CancellationToken.None); + blocked.Should().BeEmpty(); + + await _sut.ReleaseLeasedAsync(leaseToken, _now, CancellationToken.None); + + var unblocked = await _sut.GetDueJobsAsync(QueueKey.Default, _now, batchSize: 10, CancellationToken.None); + unblocked.Should().HaveCount(1); + unblocked.Single().Id.Should().Be(job1.Id); + } + // ------------------------------------------------------------------ // Helper // ------------------------------------------------------------------ From 0846a12c9e9aa3a7e371d235a4bf9ac46e8f22cb Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 14:28:53 +0200 Subject: [PATCH 18/53] test(08-01): add failing InsertAsync FIFO sequence and idempotency tests - InsertAsync_WhenPartitionedJob_ShouldAssignSequenceNumberStartingAtOne - InsertAsync_WhenPartitionedJobsInDifferentQueues_ShouldAssignIndependentSequences - InsertAsync_WhenUnpartitionedJob_ShouldLeaveSequenceNumberNull - InsertAsync_WhenIdempotencyKeyCollision_ShouldReturnExistingIdAndAssignExistingSequenceNumber - InsertAsync_WhenIdempotencyKeyCollision_ShouldNotIncreaseJobCount --- .../Storage/InMemoryStorageTests.cs | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/tests/Atomizer.Tests/Storage/InMemoryStorageTests.cs b/tests/Atomizer.Tests/Storage/InMemoryStorageTests.cs index db5801a..e6f073d 100644 --- a/tests/Atomizer.Tests/Storage/InMemoryStorageTests.cs +++ b/tests/Atomizer.Tests/Storage/InMemoryStorageTests.cs @@ -1,4 +1,5 @@ using System.Collections.Concurrent; +using Atomizer; using Atomizer.Core; using Atomizer.Storage; @@ -252,5 +253,97 @@ public async Task GetDueSchedulesAsync_WhenDueSchedulesExist_ShouldGetSchedules( ); schedules[schedule.JobKey].JobKey.Should().Be(schedule.JobKey); } + + // ---- FIFO-09: InsertAsync sequence number assignment ---- + + [Fact] + public async Task InsertAsync_WhenPartitionedJob_ShouldAssignSequenceNumberStartingAtOne() + { + // Arrange + var pk = new PartitionKey("order-1"); + var job1 = AtomizerJob.Create(QueueKey.Default, typeof(string), "p1", _now, _now, partitionKey: pk); + var job2 = AtomizerJob.Create(QueueKey.Default, typeof(string), "p2", _now, _now, partitionKey: pk); + + // Act + await _sut.InsertAsync(job1, CancellationToken.None); + await _sut.InsertAsync(job2, CancellationToken.None); + + // Assert + job1.SequenceNumber.Should().Be(1L); + job2.SequenceNumber.Should().Be(2L); + } + + [Fact] + public async Task InsertAsync_WhenPartitionedJobsInDifferentQueues_ShouldAssignIndependentSequences() + { + // Arrange + var pk = new PartitionKey("shared-key"); + var queueA = QueueKey.Default; + var queueB = new QueueKey("queue-b"); + var jobA = AtomizerJob.Create(queueA, typeof(string), "pa", _now, _now, partitionKey: pk); + var jobB = AtomizerJob.Create(queueB, typeof(string), "pb", _now, _now, partitionKey: pk); + + // Act + await _sut.InsertAsync(jobA, CancellationToken.None); + await _sut.InsertAsync(jobB, CancellationToken.None); + + // Assert — each queue starts its own sequence at 1 + jobA.SequenceNumber.Should().Be(1L); + jobB.SequenceNumber.Should().Be(1L); + } + + [Fact] + public async Task InsertAsync_WhenUnpartitionedJob_ShouldLeaveSequenceNumberNull() + { + // Arrange + var job = AtomizerJob.Create(QueueKey.Default, typeof(string), "p", _now, _now); + + // Act + await _sut.InsertAsync(job, CancellationToken.None); + + // Assert + job.SequenceNumber.Should().BeNull(); + } + + [Fact] + public async Task InsertAsync_WhenIdempotencyKeyCollision_ShouldReturnExistingIdAndAssignExistingSequenceNumber() + { + // Arrange + var pk = new PartitionKey("idem-pk"); + const string idemKey = "test-idem-key"; + var job1 = AtomizerJob.Create(QueueKey.Default, typeof(string), "p1", _now, _now, idempotencyKey: idemKey, partitionKey: pk); + await _sut.InsertAsync(job1, CancellationToken.None); + + var job2 = AtomizerJob.Create(QueueKey.Default, typeof(string), "p2", _now, _now, idempotencyKey: idemKey, partitionKey: pk); + + // Act + var returnedId = await _sut.InsertAsync(job2, CancellationToken.None); + + // Assert + returnedId.Should().Be(job1.Id); + job2.SequenceNumber.Should().Be(job1.SequenceNumber); + } + + [Fact] + public async Task InsertAsync_WhenIdempotencyKeyCollision_ShouldNotIncreaseJobCount() + { + // Arrange + const string idemKey = "idem-count-key"; + var pk = new PartitionKey("count-pk"); + var job1 = AtomizerJob.Create(QueueKey.Default, typeof(string), "p1", _now, _now, idempotencyKey: idemKey, partitionKey: pk); + await _sut.InsertAsync(job1, CancellationToken.None); + + var job2 = AtomizerJob.Create(QueueKey.Default, typeof(string), "p2", _now, _now, idempotencyKey: idemKey, partitionKey: pk); + + // Act + await _sut.InsertAsync(job2, CancellationToken.None); + + // Assert + var jobs = NonPublicSpy.GetFieldValue>( + "_jobs", + _sut + ); + jobs.Count.Should().Be(1); + } } } From 1d7588a96d14954db0229a8d003c3f6bad29f0f3 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 14:29:32 +0200 Subject: [PATCH 19/53] feat(08-01): implement FIFO InsertAsync with idempotency fix and sequence assignment - Add _partitionSequences field (ConcurrentDictionary per queue per partition) - CR-01: idempotency check before sequence assignment - returns existing Id - FIFO-09: assign monotonically increasing SequenceNumber per (queue, partitionKey) - Unpartitioned jobs leave SequenceNumber null - On collision: assigns existing.SequenceNumber to passed-in job, skips insert --- src/Atomizer/Storage/InMemoryStorage.cs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/Atomizer/Storage/InMemoryStorage.cs b/src/Atomizer/Storage/InMemoryStorage.cs index d9bc317..b4ae6bf 100644 --- a/src/Atomizer/Storage/InMemoryStorage.cs +++ b/src/Atomizer/Storage/InMemoryStorage.cs @@ -12,6 +12,7 @@ public sealed class InMemoryStorage : IAtomizerStorage { private readonly ConcurrentDictionary _jobs = new(); private readonly ConcurrentDictionary> _queues = new(); + private readonly ConcurrentDictionary> _partitionSequences = new(); private readonly ConcurrentDictionary> _leasesByToken = new(); private readonly Dictionary _schedules = new(); @@ -39,8 +40,27 @@ public Task InsertAsync(AtomizerJob job, CancellationToken cancellationTok { cancellationToken.ThrowIfCancellationRequested(); - _jobs[job.Id] = job; + // 1) CR-01 idempotency check — linear scan is acceptable for in-process storage + if (job.IdempotencyKey != null) + { + var existing = _jobs.Values.FirstOrDefault(j => j.IdempotencyKey == job.IdempotencyKey); + if (existing != null) + { + job.SequenceNumber = existing.SequenceNumber; + return Task.FromResult(existing.Id); + } + } + // 2) FIFO-09 sequence assignment — only for partitioned, non-duplicate jobs + if (job.PartitionKey != null) + { + var partitionSequences = _partitionSequences.GetOrAdd(job.QueueKey, _ => new ConcurrentDictionary()); + var seq = partitionSequences.AddOrUpdate(job.PartitionKey.Key, 1L, (_, current) => current + 1L); + job.SequenceNumber = seq; + } + + // 3) Store + index (unchanged) + _jobs[job.Id] = job; IndexIntoQueue(job); _logger.LogDebug( @@ -51,7 +71,6 @@ public Task InsertAsync(AtomizerJob job, CancellationToken cancellationTok ); EvictCompletedAndFailed(); - return Task.FromResult(job.Id); } From 2abf01db470d2d792e92dd7fc03c785b5206c36d Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 14:30:21 +0200 Subject: [PATCH 20/53] test(08-01): add failing GetDueJobsAsync FIFO partition blocking tests - GetDueJobsAsync_WhenTwoJobsSharePartition_ShouldReturnOnlyLowestSequenceNumber - GetDueJobsAsync_WhenPartitionHeadAndUnpartitionedJobExist_ShouldReturnBoth - GetDueJobsAsync_WhenPartitionJobIsProcessing_ShouldReturnEmpty - GetDueJobsAsync_WhenPartitionJobIsPendingWithAttempts_ShouldReturnEmpty - GetDueJobsAsync_WhenQueueABlockedPartitionSameKeyAsQueueB_ShouldReturnQueueBJobUnaffected - GetDueJobsAsync_WhenProcessingJobHasExpiredVisibleAt_ShouldReturnIt --- .../Storage/InMemoryStorageTests.cs | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/tests/Atomizer.Tests/Storage/InMemoryStorageTests.cs b/tests/Atomizer.Tests/Storage/InMemoryStorageTests.cs index e6f073d..50dd6cc 100644 --- a/tests/Atomizer.Tests/Storage/InMemoryStorageTests.cs +++ b/tests/Atomizer.Tests/Storage/InMemoryStorageTests.cs @@ -345,5 +345,125 @@ public async Task InsertAsync_WhenIdempotencyKeyCollision_ShouldNotIncreaseJobCo ); jobs.Count.Should().Be(1); } + + // ---- FIFO-07/FIFO-08: GetDueJobsAsync partition blocking ---- + + [Fact] + public async Task GetDueJobsAsync_WhenTwoJobsSharePartition_ShouldReturnOnlyLowestSequenceNumber() + { + // Arrange + var pk = new PartitionKey("fifo-batch"); + var job1 = AtomizerJob.Create(QueueKey.Default, typeof(string), "p1", _now, _now, partitionKey: pk); + var job2 = AtomizerJob.Create(QueueKey.Default, typeof(string), "p2", _now, _now, partitionKey: pk); + await _sut.InsertAsync(job1, CancellationToken.None); + await _sut.InsertAsync(job2, CancellationToken.None); + + // Act + var result = await _sut.GetDueJobsAsync(QueueKey.Default, _now, 10, CancellationToken.None); + + // Assert — only head of partition returned + result.Should().HaveCount(1); + result[0].Id.Should().Be(job1.Id); + } + + [Fact] + public async Task GetDueJobsAsync_WhenPartitionHeadAndUnpartitionedJobExist_ShouldReturnBoth() + { + // Arrange + var pk = new PartitionKey("mixed-pk"); + var partitioned = AtomizerJob.Create(QueueKey.Default, typeof(string), "pp", _now, _now, partitionKey: pk); + var unpartitioned = AtomizerJob.Create(QueueKey.Default, typeof(string), "up", _now, _now); + await _sut.InsertAsync(partitioned, CancellationToken.None); + await _sut.InsertAsync(unpartitioned, CancellationToken.None); + + // Act + var result = await _sut.GetDueJobsAsync(QueueKey.Default, _now, 10, CancellationToken.None); + + // Assert — both returned + result.Should().HaveCount(2); + result.Should().Contain(j => j.Id == partitioned.Id); + result.Should().Contain(j => j.Id == unpartitioned.Id); + } + + [Fact] + public async Task GetDueJobsAsync_WhenPartitionJobIsProcessing_ShouldReturnEmpty() + { + // Arrange + var pk = new PartitionKey("blocked-pk"); + var job1 = AtomizerJob.Create(QueueKey.Default, typeof(string), "p1", _now, _now, partitionKey: pk); + await _sut.InsertAsync(job1, CancellationToken.None); + var leaseToken = new LeaseToken("inst:*:default:*:lease1"); + job1.Lease(leaseToken, _now, TimeSpan.FromMinutes(10)); + await _sut.UpdateJobsAsync([job1], CancellationToken.None); + + // Act + var result = await _sut.GetDueJobsAsync(QueueKey.Default, _now, 10, CancellationToken.None); + + // Assert — partition blocked while job is Processing + result.Should().BeEmpty(); + } + + [Fact] + public async Task GetDueJobsAsync_WhenPartitionJobIsPendingWithAttempts_ShouldReturnEmpty() + { + // Arrange + var pk = new PartitionKey("retry-pk"); + var job1 = AtomizerJob.Create(QueueKey.Default, typeof(string), "p1", _now, _now, partitionKey: pk); + await _sut.InsertAsync(job1, CancellationToken.None); + // Simulate retry state: Lease → Attempt → Reschedule (Pending with Attempts = 1) + var leaseToken = new LeaseToken("inst:*:default:*:lease2"); + job1.Lease(leaseToken, _now, TimeSpan.FromMinutes(10)); + job1.Attempt(); + job1.Reschedule(_now, _now); + await _sut.UpdateJobsAsync([job1], CancellationToken.None); + + // Act + var result = await _sut.GetDueJobsAsync(QueueKey.Default, _now, 10, CancellationToken.None); + + // Assert — partition blocked while job is Pending with Attempts > 0 + result.Should().BeEmpty(); + } + + [Fact] + public async Task GetDueJobsAsync_WhenQueueABlockedPartitionSameKeyAsQueueB_ShouldReturnQueueBJobUnaffected() + { + // Arrange + var pk = new PartitionKey("cross-queue-pk"); + var queueB = new QueueKey("queue-b-test"); + var jobA = AtomizerJob.Create(QueueKey.Default, typeof(string), "pa", _now, _now, partitionKey: pk); + var jobB = AtomizerJob.Create(queueB, typeof(string), "pb", _now, _now, partitionKey: pk); + await _sut.InsertAsync(jobA, CancellationToken.None); + await _sut.InsertAsync(jobB, CancellationToken.None); + // Block partition in queue A + var leaseToken = new LeaseToken("inst:*:default:*:lease3"); + jobA.Lease(leaseToken, _now, TimeSpan.FromMinutes(10)); + await _sut.UpdateJobsAsync([jobA], CancellationToken.None); + + // Act — query queue B + var result = await _sut.GetDueJobsAsync(queueB, _now, 10, CancellationToken.None); + + // Assert — queue B is unaffected + result.Should().HaveCount(1); + result[0].Id.Should().Be(jobB.Id); + } + + [Fact] + public async Task GetDueJobsAsync_WhenProcessingJobHasExpiredVisibleAt_ShouldReturnIt() + { + // Arrange — Processing job with VisibleAt in the past (expired lease) + var job = AtomizerJob.Create(QueueKey.Default, typeof(string), "p", _now, _now); + await _sut.InsertAsync(job, CancellationToken.None); + var expiredNow = _now.AddMinutes(-10); + var leaseToken = new LeaseToken("inst:*:default:*:lease4"); + job.Lease(leaseToken, expiredNow, TimeSpan.FromMinutes(1)); // VisibleAt = expiredNow + 1min = _now - 9min + await _sut.UpdateJobsAsync([job], CancellationToken.None); + + // Act — query at _now (VisibleAt is in the past) + var result = await _sut.GetDueJobsAsync(QueueKey.Default, _now, 10, CancellationToken.None); + + // Assert — expired lease job is still returned (existing behavior must not regress) + result.Should().HaveCount(1); + result[0].Id.Should().Be(job.Id); + } } } From aa161ea213393f9005b17f5a7c4f12f20a033730 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 14:30:52 +0200 Subject: [PATCH 21/53] feat(08-01): implement FIFO GetDueJobsAsync with partition blocking filter - Pass 1: collect blocked partition keys scanning full queue snapshot - Pass 2: filter eligible candidates excluding blocked partitions - Pass 3: FIFO head-of-partition selection via OrderBy(SequenceNumber).First() - netstandard2.0-safe: OrderBy().First() instead of MinBy() - Unpartitioned jobs returned alongside partition heads --- src/Atomizer/Storage/InMemoryStorage.cs | 27 +++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/Atomizer/Storage/InMemoryStorage.cs b/src/Atomizer/Storage/InMemoryStorage.cs index b4ae6bf..acd3f97 100644 --- a/src/Atomizer/Storage/InMemoryStorage.cs +++ b/src/Atomizer/Storage/InMemoryStorage.cs @@ -123,7 +123,18 @@ CancellationToken cancellationToken return Task.FromResult((IReadOnlyList)Array.Empty()); } - candidates = ids + // Pass 1: collect blocked partition keys from the FULL queue snapshot + // D-04: blocking check must scan ALL jobs in the queue, not just due-time candidates + // A partition is blocked if any job has IsPartitionBlocked == true (Processing OR Pending+Attempts>0) + var blockedPartitions = new HashSet(); + foreach (var id in ids.Keys) + { + if (_jobs.TryGetValue(id, out var bj) && bj.IsPartitionBlocked) + blockedPartitions.Add(bj.PartitionKey!.Key); + } + + // Pass 2: filter eligible candidates (existing status+time logic) and exclude blocked partitions + var eligible = ids .Keys.Select(id => _jobs.TryGetValue(id, out var j) ? j : null) .Where(j => j != null @@ -134,8 +145,20 @@ CancellationToken cancellationToken && j.ScheduledAt <= now ) || (j.Status == AtomizerJobStatus.Processing && j.VisibleAt <= now) // expired lease ) + && (j.PartitionKey == null || !blockedPartitions.Contains(j.PartitionKey.Key)) ) - .Select(j => j!) + .Select(j => j!); + + // Pass 3: FIFO head-of-partition selection + // Use OrderBy().First() not MinBy() — MinBy is .NET 6+ and src/Atomizer targets netstandard2.0 + var unpartitioned = eligible.Where(j => j.PartitionKey == null); + var partitionHeads = eligible + .Where(j => j.PartitionKey != null) + .GroupBy(j => j.PartitionKey!.Key) + .Select(g => g.OrderBy(j => j.SequenceNumber).First()); + + candidates = unpartitioned + .Concat(partitionHeads) .OrderBy(j => j.ScheduledAt) .ThenBy(j => j.CreatedAt) .Take(Math.Max(0, batchSize)) From 691d71a85d77ad2a5b0940f68f2b1c29ed8c644d Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 14:31:57 +0200 Subject: [PATCH 22/53] feat(08-01): wire ScheduleProcessor to forward PartitionKey into AtomizerJob.Create - Pass partitionKey: schedule.PartitionKey as final named argument - Ensures scheduled jobs inherit partition key from the recurring schedule - Format InMemoryStorage.cs to comply with CSharpier (printWidth: 120) --- src/Atomizer/Scheduling/ScheduleProcessor.cs | 3 ++- src/Atomizer/Storage/InMemoryStorage.cs | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Atomizer/Scheduling/ScheduleProcessor.cs b/src/Atomizer/Scheduling/ScheduleProcessor.cs index 6073a42..c8097c6 100644 --- a/src/Atomizer/Scheduling/ScheduleProcessor.cs +++ b/src/Atomizer/Scheduling/ScheduleProcessor.cs @@ -46,7 +46,8 @@ public async Task ProcessAsync(AtomizerSchedule schedule, DateTimeOffset horizon occurrence, schedule.RetryStrategy, idempotencyKey, - schedule.JobKey + schedule.JobKey, + partitionKey: schedule.PartitionKey ); try diff --git a/src/Atomizer/Storage/InMemoryStorage.cs b/src/Atomizer/Storage/InMemoryStorage.cs index acd3f97..f5f3aea 100644 --- a/src/Atomizer/Storage/InMemoryStorage.cs +++ b/src/Atomizer/Storage/InMemoryStorage.cs @@ -54,7 +54,10 @@ public Task InsertAsync(AtomizerJob job, CancellationToken cancellationTok // 2) FIFO-09 sequence assignment — only for partitioned, non-duplicate jobs if (job.PartitionKey != null) { - var partitionSequences = _partitionSequences.GetOrAdd(job.QueueKey, _ => new ConcurrentDictionary()); + var partitionSequences = _partitionSequences.GetOrAdd( + job.QueueKey, + _ => new ConcurrentDictionary() + ); var seq = partitionSequences.AddOrUpdate(job.PartitionKey.Key, 1L, (_, current) => current + 1L); job.SequenceNumber = seq; } From 23f0710666ae20d4c8e27d59996f2a19211efc7d Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 14:33:18 +0200 Subject: [PATCH 23/53] docs(08-01): complete FIFO InMemoryStorage implementation plan summary - FIFO InsertAsync with CR-01 idempotency fix and sequence assignment - FIFO GetDueJobsAsync three-pass partition blocking filter - ScheduleProcessor partitionKey forwarding - 11 new unit tests added (TDD: RED + GREEN gates satisfied) - 104/104 tests passing --- .../08-01-SUMMARY.md | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 .planning/phases/08-inmemory-backend-unit-tests/08-01-SUMMARY.md diff --git a/.planning/phases/08-inmemory-backend-unit-tests/08-01-SUMMARY.md b/.planning/phases/08-inmemory-backend-unit-tests/08-01-SUMMARY.md new file mode 100644 index 0000000..5aa4a2a --- /dev/null +++ b/.planning/phases/08-inmemory-backend-unit-tests/08-01-SUMMARY.md @@ -0,0 +1,125 @@ +--- +phase: 08-inmemory-backend-unit-tests +plan: "01" +subsystem: storage +tags: [fifo, inmemory, idempotency, partition-blocking, tdd] +dependency_graph: + requires: [] + provides: [InMemoryStorage-FIFO, ScheduleProcessor-PartitionKey] + affects: [08-02-PLAN.md] +tech_stack: + added: [] + patterns: + - ConcurrentDictionary nested per-queue partition sequence counter + - Three-pass FIFO filter in GetDueJobsAsync (blockedPartitions + eligible + partitionHeads) + - netstandard2.0-safe OrderBy().First() instead of MinBy() +key_files: + created: [] + modified: + - src/Atomizer/Storage/InMemoryStorage.cs + - src/Atomizer/Scheduling/ScheduleProcessor.cs + - tests/Atomizer.Tests/Storage/InMemoryStorageTests.cs +decisions: + - CR-01 idempotency check placed before sequence assignment so no sequence number is consumed for a duplicate + - Used ConcurrentDictionary> for per-(queue, partitionKey) sequences + - Used job.PartitionKey.Key (string) as inner dict key to avoid value-object boxing + - OrderBy(SequenceNumber).First() used instead of MinBy() for netstandard2.0 compatibility +metrics: + duration: "4m 21s" + completed: "2026-05-04" + tasks_completed: 3 + files_modified: 3 +--- + +# Phase 8 Plan 1: FIFO InMemoryStorage Implementation Summary + +**One-liner:** FIFO sequence assignment and partition blocking in InMemoryStorage with full CR-01 idempotency fix, plus ScheduleProcessor partitionKey forwarding. + +## What Was Built + +Implemented FIFO ordering and partition blocking in `InMemoryStorage` (satisfying FIFO-10), and wired `ScheduleProcessor` to pass `schedule.PartitionKey` into `AtomizerJob.Create()` (D-07). + +### Task 1: InsertAsync FIFO with idempotency fix (TDD) + +Added `_partitionSequences` field (`ConcurrentDictionary>`) alongside `_queues`. Replaced the flat `InsertAsync` body with a three-step implementation: + +1. **CR-01 idempotency check** — linear scan of `_jobs.Values` for a matching `IdempotencyKey`. On collision: assigns `existing.SequenceNumber` to the passed-in job and returns `existing.Id` without touching `_queues`, `_leasesByToken`, or `EvictCompletedAndFailed`. +2. **FIFO-09 sequence assignment** — `_partitionSequences.GetOrAdd` per queue, then `AddOrUpdate` for atomic increment per partition key. Sequence starts at 1. Unpartitioned jobs (`PartitionKey == null`) leave `SequenceNumber` null. +3. **Store + index** — unchanged from original. + +### Task 2: GetDueJobsAsync three-pass FIFO filter (TDD) + +Replaced the single-pass LINQ chain with three passes: + +- **Pass 1:** Build `HashSet blockedPartitions` by scanning all `ids.Keys` for jobs where `IsPartitionBlocked == true`. Uses the domain property rather than re-implementing the condition. +- **Pass 2:** Filter eligible candidates using the existing status/time logic, plus exclude jobs whose `PartitionKey.Key` is in `blockedPartitions`. +- **Pass 3:** Split eligible into `unpartitioned` and `partitionHeads`. Partition heads selected via `GroupBy(PartitionKey.Key).Select(g => g.OrderBy(SequenceNumber).First())` — netstandard2.0-safe alternative to `MinBy()`. Concat both, order by `ScheduledAt`/`CreatedAt`, take `batchSize`. + +### Task 3: ScheduleProcessor PartitionKey forwarding + +Added `partitionKey: schedule.PartitionKey` as the final named argument to `AtomizerJob.Create()` in `ScheduleProcessor.ProcessAsync`. One-line change. + +## Deviations from Plan + +### Auto-fixed Issues + +**1. [Rule 1 - Bug] CSharpier formatting violation in InMemoryStorage.cs** +- **Found during:** Task 3 verification (CSharpier check) +- **Issue:** Two long lines in `InsertAsync` (GetOrAdd and AddOrUpdate calls) exceeded printWidth: 120 +- **Fix:** Ran `csharpier format` on the file; wrapped GetOrAdd call across 3 lines +- **Files modified:** `src/Atomizer/Storage/InMemoryStorage.cs` +- **Commit:** 691d71a + +## TDD Gate Compliance + +| Task | RED commit | GREEN commit | +|------|-----------|-------------| +| Task 1 (InsertAsync) | 0846a12 — 5 failing tests | 1d7588a — all pass | +| Task 2 (GetDueJobsAsync) | 2abf01d — 2 failing tests | aa161ea — all pass | + +Both RED and GREEN gates satisfied. RED commits preceded GREEN commits. + +## Test Coverage + +11 new unit tests added to `InMemoryStorageTests.cs`: + +**InsertAsync FIFO tests (Task 1):** +- `InsertAsync_WhenPartitionedJob_ShouldAssignSequenceNumberStartingAtOne` +- `InsertAsync_WhenPartitionedJobsInDifferentQueues_ShouldAssignIndependentSequences` +- `InsertAsync_WhenUnpartitionedJob_ShouldLeaveSequenceNumberNull` +- `InsertAsync_WhenIdempotencyKeyCollision_ShouldReturnExistingIdAndAssignExistingSequenceNumber` +- `InsertAsync_WhenIdempotencyKeyCollision_ShouldNotIncreaseJobCount` + +**GetDueJobsAsync FIFO tests (Task 2):** +- `GetDueJobsAsync_WhenTwoJobsSharePartition_ShouldReturnOnlyLowestSequenceNumber` +- `GetDueJobsAsync_WhenPartitionHeadAndUnpartitionedJobExist_ShouldReturnBoth` +- `GetDueJobsAsync_WhenPartitionJobIsProcessing_ShouldReturnEmpty` +- `GetDueJobsAsync_WhenPartitionJobIsPendingWithAttempts_ShouldReturnEmpty` +- `GetDueJobsAsync_WhenQueueABlockedPartitionSameKeyAsQueueB_ShouldReturnQueueBJobUnaffected` +- `GetDueJobsAsync_WhenProcessingJobHasExpiredVisibleAt_ShouldReturnIt` + +## Verification Results + +``` +dotnet build — 0 errors (pre-existing NU1903 warnings only) +dotnet test tests/Atomizer.Tests — 104/104 passed (net8.0 + net10.0) +csharpier check — all files formatted +``` + +## Known Stubs + +None — all data paths are fully wired. + +## Threat Flags + +None — all changes are internal in-process storage; no new network endpoints, public API surface, or trust boundary crossings introduced. + +## Self-Check: PASSED + +- `src/Atomizer/Storage/InMemoryStorage.cs` contains `_partitionSequences` field: confirmed +- `InsertAsync` contains `_jobs.Values.FirstOrDefault(j => j.IdempotencyKey == job.IdempotencyKey)`: confirmed +- `InsertAsync` contains `_partitionSequences.GetOrAdd` and `partitionSequences.AddOrUpdate`: confirmed +- `GetDueJobsAsync` contains `blockedPartitions` HashSet and `IsPartitionBlocked` usage: confirmed +- `GetDueJobsAsync` contains `partitionHeads` and `unpartitioned` variables: confirmed +- `ScheduleProcessor.cs` contains `partitionKey: schedule.PartitionKey`: confirmed +- Commits exist: 0846a12, 1d7588a, 2abf01d, aa161ea, 691d71a From b2ee2d2bddfc1d9530ee6bdfa6dd898fb753f688 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 14:37:30 +0200 Subject: [PATCH 24/53] feat(08-02): add InMemoryStorageContractTests concrete subclass - Implements CreateStorage(IAtomizerClock) returning new InMemoryStorage - Inherits all 9 FIFO contract tests from AtomizerStorageContractTests - AmountOfJobsToRetainInMemory=100 ensures terminal jobs retained during tests - All 9 inherited tests pass (113/113 total suite green on net8.0) --- .../Storage/InMemoryStorageContractTests.cs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/Atomizer.Tests/Storage/InMemoryStorageContractTests.cs diff --git a/tests/Atomizer.Tests/Storage/InMemoryStorageContractTests.cs b/tests/Atomizer.Tests/Storage/InMemoryStorageContractTests.cs new file mode 100644 index 0000000..7bbe541 --- /dev/null +++ b/tests/Atomizer.Tests/Storage/InMemoryStorageContractTests.cs @@ -0,0 +1,21 @@ +using Atomizer.Abstractions; +using Atomizer.Core; +using Atomizer.Storage; +using Atomizer.Tests.Utilities.StorageContract; + +namespace Atomizer.Tests.Storage; + +/// +/// Concrete contract tests for . +/// Inherits the 8 FIFO contract tests from . +/// +public sealed class InMemoryStorageContractTests : AtomizerStorageContractTests +{ + /// + protected override IAtomizerStorage CreateStorage(IAtomizerClock clock) + { + var options = new InMemoryJobStorageOptions { AmountOfJobsToRetainInMemory = 100 }; + var logger = Substitute.For>(); + return new InMemoryStorage(options, clock, logger); + } +} From ba8b01cf17db00a2062899511c6e5096133a9ee6 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 15:23:48 +0200 Subject: [PATCH 25/53] test(08-03): add terminal-state unblocking tests (FIFO-13) - GetDueJobsAsync_WhenPartitionHeadCompleted_ShouldUnblockNextJob - GetDueJobsAsync_WhenPartitionHeadFailed_ShouldUnblockNextJob --- .../AtomizerStorageContractTests.cs | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs b/tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs index e85bd64..d55c2fb 100644 --- a/tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs +++ b/tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs @@ -271,6 +271,68 @@ public async Task ReleaseLeasedAsync_WhenPartitionHeadReleased_ShouldUnblockPart unblocked.Single().Id.Should().Be(job1.Id); } + // ------------------------------------------------------------------ + // FIFO-13: terminal-state unblocking + // ------------------------------------------------------------------ + + /// + /// FIFO-13: When the head job of a partition completes successfully, the partition + /// is unblocked and the next job becomes eligible for processing. + /// + [Fact] + public async Task GetDueJobsAsync_WhenPartitionHeadCompleted_ShouldUnblockNextJob() + { + // Arrange + var partitionKey = new PartitionKey("complete-p"); + var job1 = CreateJob(partitionKey: partitionKey); + var job2 = CreateJob(partitionKey: partitionKey); + + await _sut.InsertAsync(job1, CancellationToken.None); + await _sut.InsertAsync(job2, CancellationToken.None); + + // Transition job1 to Completed: Lease → Attempt → MarkAsCompleted + job1.Lease(FakeDataFactory.LeaseToken(), _now, TimeSpan.FromMinutes(10)); + job1.Attempt(); + job1.MarkAsCompleted(_now); + await _sut.UpdateJobsAsync([job1], CancellationToken.None); + + // Act + var result = await _sut.GetDueJobsAsync(QueueKey.Default, _now, batchSize: 10, CancellationToken.None); + + // Assert — job2 is now the partition head and must be returned + result.Should().HaveCount(1); + result.Single().Id.Should().Be(job2.Id); + } + + /// + /// FIFO-13: When the head job of a partition exhausts its retries and is marked Failed, + /// the partition is unblocked and the next job becomes eligible for processing. + /// + [Fact] + public async Task GetDueJobsAsync_WhenPartitionHeadFailed_ShouldUnblockNextJob() + { + // Arrange + var partitionKey = new PartitionKey("failed-p"); + var job1 = CreateJob(partitionKey: partitionKey); + var job2 = CreateJob(partitionKey: partitionKey); + + await _sut.InsertAsync(job1, CancellationToken.None); + await _sut.InsertAsync(job2, CancellationToken.None); + + // Transition job1 to Failed: Lease → Attempt → MarkAsFailed + job1.Lease(FakeDataFactory.LeaseToken(), _now, TimeSpan.FromMinutes(10)); + job1.Attempt(); + job1.MarkAsFailed(_now); + await _sut.UpdateJobsAsync([job1], CancellationToken.None); + + // Act + var result = await _sut.GetDueJobsAsync(QueueKey.Default, _now, batchSize: 10, CancellationToken.None); + + // Assert — job2 is now the partition head and must be returned + result.Should().HaveCount(1); + result.Single().Id.Should().Be(job2.Id); + } + // ------------------------------------------------------------------ // Helper // ------------------------------------------------------------------ From dc7a024f9698d5fc6496321fb21b671312470a56 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 15:24:58 +0200 Subject: [PATCH 26/53] docs(08-03): complete terminal-state unblocking tests plan summary --- .../08-03-SUMMARY.md | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .planning/phases/08-inmemory-backend-unit-tests/08-03-SUMMARY.md diff --git a/.planning/phases/08-inmemory-backend-unit-tests/08-03-SUMMARY.md b/.planning/phases/08-inmemory-backend-unit-tests/08-03-SUMMARY.md new file mode 100644 index 0000000..a7ac7f7 --- /dev/null +++ b/.planning/phases/08-inmemory-backend-unit-tests/08-03-SUMMARY.md @@ -0,0 +1,81 @@ +--- +phase: 08-inmemory-backend-unit-tests +plan: "03" +subsystem: tests +tags: [fifo, contract-tests, terminal-state, unblocking, fifo-13] +dependency_graph: + requires: [08-02-PLAN.md] + provides: [FIFO-13 terminal-state unblocking coverage] + affects: [AtomizerStorageContractTests, InMemoryStorageContractTests] +tech_stack: + added: [] + patterns: [xUnit Fact, domain-method-chain (Lease→Attempt→MarkAsCompleted/MarkAsFailed)] +key_files: + created: [] + modified: + - tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs +decisions: + - "Used domain method chain (Lease→Attempt→MarkAsCompleted/MarkAsFailed) in test body to transition job1 to terminal state, consistent with existing test patterns" +metrics: + duration: "~4 minutes" + completed: "2026-05-04" + tasks_completed: 1 + tasks_total: 1 + files_changed: 1 +--- + +# Phase 08 Plan 03: Terminal-State Unblocking Tests Summary + +Two contract tests added to AtomizerStorageContractTests covering FIFO-13 terminal-state unblocking: Completed and Failed head jobs release their partition for the next job. + +## Tasks Completed + +| Task | Name | Commit | Files | +|------|------|--------|-------| +| 1 | Add terminal-unblocking tests to AtomizerStorageContractTests | ba8b01c | tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs | + +## What Was Built + +Added two `[Fact]` methods to `AtomizerStorageContractTests` under a `// FIFO-13: terminal-state unblocking` comment block, inserted immediately before the `// Helper` region: + +1. **`GetDueJobsAsync_WhenPartitionHeadCompleted_ShouldUnblockNextJob`** — inserts two partitioned jobs, transitions job1 to `Completed` via `Lease → Attempt → MarkAsCompleted`, calls `UpdateJobsAsync`, then asserts `GetDueJobsAsync` returns job2. + +2. **`GetDueJobsAsync_WhenPartitionHeadFailed_ShouldUnblockNextJob`** — same structure but transitions job1 to `Failed` via `Lease → Attempt → MarkAsFailed`. + +Both tests are inherited by `InMemoryStorageContractTests` (and all future backend contract subclasses) without any additional code changes. + +## Verification Results + +- `InMemoryStorageContractTests` now runs **11 tests** (was 9 before this plan). +- **net8.0**: 11/11 passed. +- **net10.0**: 11/11 passed. +- net6.0 test host failed to launch (pre-existing environment issue — .NET 6 runtime not installed on this machine; 0 build errors). +- Both new tests confirmed passing in both supported runtimes. + +Acceptance criteria: + +``` +grep -c "GetDueJobsAsync_WhenPartitionHeadCompleted_ShouldUnblockNextJob\|GetDueJobsAsync_WhenPartitionHeadFailed_ShouldUnblockNextJob" → 2 +grep -c "MarkAsCompleted\|MarkAsFailed" → 4 +``` + +Both output as expected. + +## Deviations from Plan + +None — plan executed exactly as written. + +## Known Stubs + +None. + +## Threat Flags + +None — test utilities file only; no new production code, network endpoints, auth paths, or schema changes introduced. + +## Self-Check: PASSED + +- [x] `tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs` modified — verified +- [x] Commit `ba8b01c` exists — verified +- [x] Both new test methods present (grep count = 2) — verified +- [x] All 11 InMemoryStorageContractTests pass on net8.0 and net10.0 — verified From 9135e93c8bb9dc4749fd1f3ac6b02dfe7b161f20 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 16:40:54 +0200 Subject: [PATCH 27/53] fix(ef-core-tests): lease jobs before marking completed/failed in UpdateJobsAsync test Co-Authored-By: Claude Sonnet 4.6 --- .../Storage/EntityFrameworkCoreStorageTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/EntityFrameworkCoreStorageTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/EntityFrameworkCoreStorageTests.cs index 959d49e..3e29963 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/EntityFrameworkCoreStorageTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/EntityFrameworkCoreStorageTests.cs @@ -127,6 +127,8 @@ public async Task UpdateJobsAsync_WhenJobsExist_ShouldUpdateJobs() dbContext.ChangeTracker.Clear(); // Act + job1.Lease(FakeDataFactory.LeaseToken(), _clock.UtcNow, TimeSpan.FromMinutes(10)); + job2.Lease(FakeDataFactory.LeaseToken(), _clock.UtcNow, TimeSpan.FromMinutes(10)); job1.MarkAsCompleted(_clock.UtcNow); job2.MarkAsFailed(_clock.UtcNow); await storage.UpdateJobsAsync(new[] { job1, job2 }, CancellationToken.None); From a37afb8145b5a97cf070e200f63edf070283ade2 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 17:21:25 +0200 Subject: [PATCH 28/53] chore(tests): update AwesomeAssertions package version to 9.4.0 --- .../Atomizer.EntityFrameworkCore.Tests.csproj | 1 - .../Atomizer.Tests.Utilities/Atomizer.Tests.Utilities.csproj | 4 ++-- tests/Atomizer.Tests/Atomizer.Tests.csproj | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Atomizer.EntityFrameworkCore.Tests.csproj b/tests/Atomizer.EntityFrameworkCore.Tests/Atomizer.EntityFrameworkCore.Tests.csproj index a98529d..e114e14 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Atomizer.EntityFrameworkCore.Tests.csproj +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Atomizer.EntityFrameworkCore.Tests.csproj @@ -26,7 +26,6 @@ - all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/Atomizer.Tests.Utilities/Atomizer.Tests.Utilities.csproj b/tests/Atomizer.Tests.Utilities/Atomizer.Tests.Utilities.csproj index 2cf2465..a5bdd91 100644 --- a/tests/Atomizer.Tests.Utilities/Atomizer.Tests.Utilities.csproj +++ b/tests/Atomizer.Tests.Utilities/Atomizer.Tests.Utilities.csproj @@ -1,4 +1,4 @@ - + net6.0;net8.0;net10.0 enable @@ -11,7 +11,7 @@ - + diff --git a/tests/Atomizer.Tests/Atomizer.Tests.csproj b/tests/Atomizer.Tests/Atomizer.Tests.csproj index 01ea208..06d1fea 100644 --- a/tests/Atomizer.Tests/Atomizer.Tests.csproj +++ b/tests/Atomizer.Tests/Atomizer.Tests.csproj @@ -27,7 +27,6 @@ - all runtime; build; native; contentfiles; analyzers; buildtransitive From 4e792dae2b5b0b0ccba254dc6183d11c1f096613 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 18:01:09 +0200 Subject: [PATCH 29/53] feat(09-01): add PartitionKey and SequenceNumber to AtomizerJobEntity - Add string? PartitionKey and long? SequenceNumber properties with XML docs - Map job.PartitionKey?.ToString() and job.SequenceNumber in ToEntity - Map entity.PartitionKey back to PartitionKey? value object in ToAtomizerJob --- .../Entities/AtomizerJobEntity.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs b/src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs index 08160d8..5736f08 100644 --- a/src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs +++ b/src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs @@ -53,6 +53,12 @@ public class AtomizerJobEntity /// Gets or sets the idempotency key used to deduplicate job insertions. public string? IdempotencyKey { get; set; } + /// Gets or sets the partition key grouping this job for FIFO processing, or null if unpartitioned. + public string? PartitionKey { get; set; } + + /// Gets or sets the monotonically increasing sequence number within (queue, partition key), or null if unpartitioned. + public long? SequenceNumber { get; set; } + /// Gets or sets the list of error records from previous failed attempts. public List Errors { get; set; } = new List(); } @@ -105,6 +111,8 @@ public static AtomizerJobEntity ToEntity(this AtomizerJob job) RetryIntervals = job.RetryStrategy.RetryIntervals, ScheduleJobKey = job.ScheduleJobKey?.ToString(), IdempotencyKey = job.IdempotencyKey, + PartitionKey = job.PartitionKey?.ToString(), + SequenceNumber = job.SequenceNumber, Errors = job.Errors.Select(err => err.ToEntity()).ToList(), }; } @@ -135,6 +143,8 @@ public static AtomizerJob ToAtomizerJob(this AtomizerJobEntity entity) entity.RetryIntervals.Length == 0 ? RetryStrategy.None : RetryStrategy.Intervals(entity.RetryIntervals), ScheduleJobKey = entity.ScheduleJobKey != null ? new JobKey(entity.ScheduleJobKey) : null, IdempotencyKey = entity.IdempotencyKey, + PartitionKey = entity.PartitionKey != null ? new PartitionKey(entity.PartitionKey) : null, + SequenceNumber = entity.SequenceNumber, Errors = entity.Errors.Select(err => err.ToAtomizerJobError()).ToList(), }; } From 50040963086b09495e3dee7c6e9df8b316efb748 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 18:01:55 +0200 Subject: [PATCH 30/53] feat(09-01): configure PartitionKey/SequenceNumber columns and add InsertJobWithSequence to ISqlDialect - Configure PartitionKey as nullable varchar(255) in AtomizerJobEntityConfiguration - Configure SequenceNumber as nullable bigint (long?) in entity config - Add FormattableString InsertJobWithSequence(AtomizerJob job) to ISqlDialect --- .../Configurations/AtomizerJobEntityConfiguration.cs | 2 ++ src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs b/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs index 62c41b3..eefd2f5 100644 --- a/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs +++ b/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs @@ -60,5 +60,7 @@ public void Configure(EntityTypeBuilder builder) c => c.ToArray() ) ); + builder.Property(job => job.PartitionKey).HasMaxLength(255).IsRequired(false); + builder.Property(job => job.SequenceNumber).IsRequired(false); } } diff --git a/src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs index 5a13e9c..e2e991a 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs @@ -6,4 +6,12 @@ internal interface ISqlDialect FormattableString ReleaseLeasedJobs(LeaseToken leaseToken, DateTimeOffset now); FormattableString GetDueSchedules(DateTimeOffset now); FormattableString UpsertScheduleAsync(AtomizerSchedule schedule, DateTimeOffset now); + + /// + /// Returns provider-specific SQL that inserts a partitioned job and atomically assigns + /// a monotonically increasing SequenceNumber scoped to the job's (queue, partition key). + /// + /// The partitioned job to insert. must not be null. + /// A ready for ExecuteSqlInterpolatedAsync. + FormattableString InsertJobWithSequence(AtomizerJob job); } From 87e293273ae8c7c9f49519990dc12dc17afede41 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 18:03:01 +0200 Subject: [PATCH 31/53] docs(09-01): complete EF Core entity layer FIFO foundation plan summary --- .../09-01-SUMMARY.md | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 .planning/phases/09-ef-core-backend-integration-tests/09-01-SUMMARY.md diff --git a/.planning/phases/09-ef-core-backend-integration-tests/09-01-SUMMARY.md b/.planning/phases/09-ef-core-backend-integration-tests/09-01-SUMMARY.md new file mode 100644 index 0000000..e5072b8 --- /dev/null +++ b/.planning/phases/09-ef-core-backend-integration-tests/09-01-SUMMARY.md @@ -0,0 +1,95 @@ +--- +phase: 09-ef-core-backend-integration-tests +plan: "01" +subsystem: ef-core-entity-layer +tags: [ef-core, entity, fifo, partition-key, sequence-number, sql-dialect] +dependency_graph: + requires: [] + provides: + - AtomizerJobEntity.PartitionKey (string?) + - AtomizerJobEntity.SequenceNumber (long?) + - AtomizerJobEntityMapper round-trip for PartitionKey and SequenceNumber + - AtomizerJobEntityConfiguration column config for both new columns + - ISqlDialect.InsertJobWithSequence method signature + affects: + - src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs + - src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs + - src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs +tech_stack: + added: [] + patterns: + - EF Core nullable column configuration via HasMaxLength(255).IsRequired(false) + - ISqlDialect strategy interface extension for provider-specific SQL +key_files: + created: [] + modified: + - src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs + - src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs + - src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs +decisions: + - ISqlDialect.InsertJobWithSequence added as interface-only contract; dialect implementations deferred to Plan 02 (expected CS0535 build failures are intentional) + - No explicit HasColumnType on SequenceNumber — EF Core auto-maps long? to bigint on all three supported providers +metrics: + duration: "2m" + completed: "2026-05-04" + tasks_completed: 2 + tasks_total: 2 + files_changed: 3 +--- + +# Phase 09 Plan 01: EF Core Entity Layer FIFO Foundation Summary + +EF Core entity, mapper, configuration, and ISqlDialect interface extended with PartitionKey (string?) and SequenceNumber (long?) as the foundational contracts for FIFO processing. + +## Tasks Completed + +| Task | Name | Commit | Files | +|------|------|--------|-------| +| 1 | Add PartitionKey and SequenceNumber to AtomizerJobEntity and both mapper directions | 4e792da | src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs | +| 2 | Configure new columns in AtomizerJobEntityConfiguration and add InsertJobWithSequence to ISqlDialect | 5004096 | src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs, src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs | + +## What Was Built + +### AtomizerJobEntity (Task 1) +- Added `public string? PartitionKey { get; set; }` with XML doc after `IdempotencyKey` +- Added `public long? SequenceNumber { get; set; }` with XML doc before `Errors` navigation property +- `ToEntity` mapper: `PartitionKey = job.PartitionKey?.ToString()` and `SequenceNumber = job.SequenceNumber` +- `ToAtomizerJob` mapper: `PartitionKey = entity.PartitionKey != null ? new PartitionKey(entity.PartitionKey) : null` and `SequenceNumber = entity.SequenceNumber` + +### AtomizerJobEntityConfiguration (Task 2) +- `builder.Property(job => job.PartitionKey).HasMaxLength(255).IsRequired(false)` — enforces same 255-char cap as the PartitionKey value object constructor (T-09-01 mitigation) +- `builder.Property(job => job.SequenceNumber).IsRequired(false)` — EF Core auto-maps long? to bigint + +### ISqlDialect (Task 2) +- Added `FormattableString InsertJobWithSequence(AtomizerJob job)` with XML doc +- Dialect implementations (PostgreSqlDialect, SqlServerDialect, MySqlDialect) will implement this in Plan 02 + +## Deviations from Plan + +None — plan executed exactly as written. The expected CS0535 build failures on all three dialect classes are confirmed and correct per the plan's verification section. + +## Verification Results + +All grep checks pass: +- `grep -c "public string? PartitionKey" AtomizerJobEntity.cs` → 1 +- `grep -c "public long? SequenceNumber" AtomizerJobEntity.cs` → 1 +- `grep -c "InsertJobWithSequence" ISqlDialect.cs` → 1 +- `grep -c "HasMaxLength(255)" AtomizerJobEntityConfiguration.cs` → 1 + +Build status: Expected CS0535 failures on MySqlDialect, PostgreSqlDialect, SqlServerDialect (dialect implementations deferred to Plan 02). + +## Known Stubs + +None — this plan establishes contracts only; no stub data flows to UI rendering. + +## Threat Surface Scan + +No new network endpoints, auth paths, file access patterns, or schema changes at trust boundaries beyond what the plan's threat model covers. + +## Self-Check: PASSED + +- 4e792da: feat(09-01): add PartitionKey and SequenceNumber to AtomizerJobEntity — FOUND +- 5004096: feat(09-01): configure PartitionKey/SequenceNumber columns and add InsertJobWithSequence to ISqlDialect — FOUND +- src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs — exists, contains `public string? PartitionKey` +- src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs — exists, contains `HasMaxLength(255)` +- src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs — exists, contains `InsertJobWithSequence` From bd51d1bbbd53a75d0ab02915f12a79c7164649bc Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 18:08:58 +0200 Subject: [PATCH 32/53] feat(09-02): implement InsertJobWithSequence and CTE-based GetDueJobs in all three dialects - Replace flat WHERE in GetDueJobs with blocked_partitions + partition_heads CTE for FIFO-aware acquisition - PostgreSQL: FOR NO KEY UPDATE SKIP LOCKED on outer SELECT, derived-table subquery in InsertJobWithSequence - SQL Server: WITH (UPDLOCK, READPAST, ROWLOCK) on outer FROM only, TOP(batchSize) via string interpolation - MySQL: LEFT JOIN anti-join in partition_heads CTE, derived-table form for COALESCE(MAX) to avoid same-table restriction - All three providers: atomic COALESCE(MAX(SequenceNumber), 0) + 1 sequence assignment per (queue, partition_key) --- .../Providers/Sql/MySqlDialect.cs | 134 ++++++++++++++++-- .../Providers/Sql/PostgreSqlDialect.cs | 133 +++++++++++++++-- .../Providers/Sql/SqlServerDialect.cs | 130 +++++++++++++++-- 3 files changed, 356 insertions(+), 41 deletions(-) diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs index 36b7e06..0dc94dd 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs @@ -23,27 +23,133 @@ public FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int b var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; var colScheduledAt = c[nameof(AtomizerJobEntity.ScheduledAt)]; var colId = c[nameof(AtomizerJobEntity.Id)]; + var colPartitionKey = c[nameof(AtomizerJobEntity.PartitionKey)]; + var colSequenceNumber = c[nameof(AtomizerJobEntity.SequenceNumber)]; + var colAttempts = c[nameof(AtomizerJobEntity.Attempts)]; var statusPending = (int)AtomizerEntityJobStatus.Pending; var statusProcessing = (int)AtomizerEntityJobStatus.Processing; var format = - $@"SELECT t.* + $@"WITH blocked_partitions AS ( + SELECT DISTINCT {colPartitionKey} + FROM {table} + WHERE {colQueueKey} = {{0}} + AND {colPartitionKey} IS NOT NULL + AND ( + {colStatus} = {statusProcessing} + OR ({colStatus} = {statusPending} AND {colAttempts} > 0) + ) +), +partition_heads AS ( + SELECT j.{colPartitionKey}, MIN(j.{colSequenceNumber}) AS min_seq + FROM {table} AS j + LEFT JOIN blocked_partitions bp ON j.{colPartitionKey} = bp.{colPartitionKey} + WHERE j.{colQueueKey} = {{1}} + AND j.{colPartitionKey} IS NOT NULL + AND bp.{colPartitionKey} IS NULL + GROUP BY j.{colPartitionKey} +) +SELECT t.* FROM {table} AS t -WHERE {colQueueKey} = {{0}} +LEFT JOIN partition_heads ph + ON t.{colPartitionKey} = ph.{colPartitionKey} + AND t.{colSequenceNumber} = ph.min_seq +WHERE t.{colQueueKey} = {{2}} AND ( - ( {colStatus} = {statusPending} - AND ( {colVisibleAt} IS NULL - OR {colVisibleAt} <= {{1}}) - AND {colScheduledAt} <= {{2}} - ) - OR - ( {colStatus} = {statusProcessing} - AND {colVisibleAt} <= {{3}} - ) + (t.{colPartitionKey} IS NULL + AND ( + ({colStatus} = {statusPending} + AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{3}}) + AND {colScheduledAt} <= {{4}}) + OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{5}}) + ) + ) + OR + (t.{colPartitionKey} IS NOT NULL AND ph.min_seq IS NOT NULL + AND ( + ({colStatus} = {statusPending} + AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{6}}) + AND {colScheduledAt} <= {{7}}) + OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{8}}) ) -ORDER BY {colScheduledAt}, {colId} -LIMIT {{4}} + ) + ) +ORDER BY t.{colScheduledAt}, t.{colId} +LIMIT {{9}} FOR UPDATE SKIP LOCKED;"; - return FormattableStringFactory.Create(format, queueKey.Key, now, now, now, batchSize); + return FormattableStringFactory.Create( + format, + queueKey.Key, + queueKey.Key, + queueKey.Key, + now, + now, + now, + now, + now, + now, + batchSize + ); + } + + public FormattableString InsertJobWithSequence(AtomizerJob job) + { + var entity = job.ToEntity(); + var table = _jobs.Table; + var c = _jobs.Col; + var colId = c[nameof(AtomizerJobEntity.Id)]; + var colQueueKey = c[nameof(AtomizerJobEntity.QueueKey)]; + var colPayloadType = c[nameof(AtomizerJobEntity.PayloadType)]; + var colPayload = c[nameof(AtomizerJobEntity.Payload)]; + var colScheduledAt = c[nameof(AtomizerJobEntity.ScheduledAt)]; + var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; + var colStatus = c[nameof(AtomizerJobEntity.Status)]; + var colAttempts = c[nameof(AtomizerJobEntity.Attempts)]; + var colRetryIntervals = c[nameof(AtomizerJobEntity.RetryIntervals)]; + var colCreatedAt = c[nameof(AtomizerJobEntity.CreatedAt)]; + var colUpdatedAt = c[nameof(AtomizerJobEntity.UpdatedAt)]; + var colLeaseToken = c[nameof(AtomizerJobEntity.LeaseToken)]; + var colScheduleJobKey = c[nameof(AtomizerJobEntity.ScheduleJobKey)]; + var colIdempotencyKey = c[nameof(AtomizerJobEntity.IdempotencyKey)]; + var colPartitionKey = c[nameof(AtomizerJobEntity.PartitionKey)]; + var colSequenceNumber = c[nameof(AtomizerJobEntity.SequenceNumber)]; + var retryIntervals = string.Join( + ";", + Array.ConvertAll(entity.RetryIntervals, ts => (long)ts.TotalMilliseconds) + ); + var format = + $@"INSERT INTO {table} ( + {colId}, {colQueueKey}, {colPayloadType}, {colPayload}, + {colScheduledAt}, {colVisibleAt}, {colStatus}, {colAttempts}, + {colRetryIntervals}, {colCreatedAt}, {colUpdatedAt}, + {colLeaseToken}, {colScheduleJobKey}, {colIdempotencyKey}, + {colPartitionKey}, {colSequenceNumber} +) +SELECT {{0}}, {{1}}, {{2}}, {{3}}, + {{4}}, {{5}}, {{6}}, {{7}}, + {{8}}, {{9}}, {{10}}, + {{11}}, {{12}}, {{13}}, + {{14}}, + COALESCE((SELECT MAX(max_seq) FROM (SELECT MAX({colSequenceNumber}) AS max_seq FROM {table} WHERE {colQueueKey} = {{15}} AND {colPartitionKey} = {{16}}) AS sub), 0) + 1;"; + return FormattableStringFactory.Create( + format, + entity.Id, + entity.QueueKey, + entity.PayloadType, + entity.Payload, + entity.ScheduledAt, + entity.VisibleAt, + (int)entity.Status, + entity.Attempts, + retryIntervals, + entity.CreatedAt, + entity.UpdatedAt, + entity.LeaseToken, + entity.ScheduleJobKey, + entity.IdempotencyKey, + entity.PartitionKey, + entity.QueueKey, + entity.PartitionKey + ); } public FormattableString ReleaseLeasedJobs(LeaseToken leaseToken, DateTimeOffset now) diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs index c1d66a2..28738d1 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs @@ -23,27 +23,132 @@ public FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int b var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; var colScheduledAt = c[nameof(AtomizerJobEntity.ScheduledAt)]; var colId = c[nameof(AtomizerJobEntity.Id)]; + var colPartitionKey = c[nameof(AtomizerJobEntity.PartitionKey)]; + var colSequenceNumber = c[nameof(AtomizerJobEntity.SequenceNumber)]; + var colAttempts = c[nameof(AtomizerJobEntity.Attempts)]; var statusPending = (int)AtomizerEntityJobStatus.Pending; var statusProcessing = (int)AtomizerEntityJobStatus.Processing; var format = - $@"SELECT t.* + $@"WITH blocked_partitions AS ( + SELECT DISTINCT {colPartitionKey} + FROM {table} + WHERE {colQueueKey} = {{0}} + AND {colPartitionKey} IS NOT NULL + AND ( + {colStatus} = {statusProcessing} + OR ({colStatus} = {statusPending} AND {colAttempts} > 0) + ) +), +partition_heads AS ( + SELECT {colPartitionKey}, MIN({colSequenceNumber}) AS min_seq + FROM {table} + WHERE {colQueueKey} = {{1}} + AND {colPartitionKey} IS NOT NULL + AND {colPartitionKey} NOT IN (SELECT {colPartitionKey} FROM blocked_partitions) + GROUP BY {colPartitionKey} +) +SELECT t.* FROM {table} AS t -WHERE {colQueueKey} = {{0}} +LEFT JOIN partition_heads ph + ON t.{colPartitionKey} = ph.{colPartitionKey} + AND t.{colSequenceNumber} = ph.min_seq +WHERE t.{colQueueKey} = {{2}} AND ( - ( {colStatus} = {statusPending} - AND ( {colVisibleAt} IS NULL - OR {colVisibleAt} <= {{1}}) - AND {colScheduledAt} <= {{2}} - ) - OR - ( {colStatus} = {statusProcessing} - AND {colVisibleAt} <= {{3}} - ) + (t.{colPartitionKey} IS NULL + AND ( + ({colStatus} = {statusPending} + AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{3}}) + AND {colScheduledAt} <= {{4}}) + OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{5}}) + ) + ) + OR + (t.{colPartitionKey} IS NOT NULL AND ph.min_seq IS NOT NULL + AND ( + ({colStatus} = {statusPending} + AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{6}}) + AND {colScheduledAt} <= {{7}}) + OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{8}}) ) -ORDER BY {colScheduledAt}, {colId} -LIMIT {{4}} + ) + ) +ORDER BY t.{colScheduledAt}, t.{colId} +LIMIT {{9}} FOR NO KEY UPDATE SKIP LOCKED;"; - return FormattableStringFactory.Create(format, queueKey.Key, now, now, now, batchSize); + return FormattableStringFactory.Create( + format, + queueKey.Key, + queueKey.Key, + queueKey.Key, + now, + now, + now, + now, + now, + now, + batchSize + ); + } + + public FormattableString InsertJobWithSequence(AtomizerJob job) + { + var entity = job.ToEntity(); + var table = _jobs.Table; + var c = _jobs.Col; + var colId = c[nameof(AtomizerJobEntity.Id)]; + var colQueueKey = c[nameof(AtomizerJobEntity.QueueKey)]; + var colPayloadType = c[nameof(AtomizerJobEntity.PayloadType)]; + var colPayload = c[nameof(AtomizerJobEntity.Payload)]; + var colScheduledAt = c[nameof(AtomizerJobEntity.ScheduledAt)]; + var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; + var colStatus = c[nameof(AtomizerJobEntity.Status)]; + var colAttempts = c[nameof(AtomizerJobEntity.Attempts)]; + var colRetryIntervals = c[nameof(AtomizerJobEntity.RetryIntervals)]; + var colCreatedAt = c[nameof(AtomizerJobEntity.CreatedAt)]; + var colUpdatedAt = c[nameof(AtomizerJobEntity.UpdatedAt)]; + var colLeaseToken = c[nameof(AtomizerJobEntity.LeaseToken)]; + var colScheduleJobKey = c[nameof(AtomizerJobEntity.ScheduleJobKey)]; + var colIdempotencyKey = c[nameof(AtomizerJobEntity.IdempotencyKey)]; + var colPartitionKey = c[nameof(AtomizerJobEntity.PartitionKey)]; + var colSequenceNumber = c[nameof(AtomizerJobEntity.SequenceNumber)]; + var retryIntervals = string.Join( + ";", + Array.ConvertAll(entity.RetryIntervals, ts => (long)ts.TotalMilliseconds) + ); + var format = + $@"INSERT INTO {table} ( + {colId}, {colQueueKey}, {colPayloadType}, {colPayload}, + {colScheduledAt}, {colVisibleAt}, {colStatus}, {colAttempts}, + {colRetryIntervals}, {colCreatedAt}, {colUpdatedAt}, + {colLeaseToken}, {colScheduleJobKey}, {colIdempotencyKey}, + {colPartitionKey}, {colSequenceNumber} +) +SELECT {{0}}, {{1}}, {{2}}, {{3}}, + {{4}}, {{5}}, {{6}}, {{7}}, + {{8}}, {{9}}, {{10}}, + {{11}}, {{12}}, {{13}}, + {{14}}, + COALESCE((SELECT MAX({colSequenceNumber}) FROM (SELECT {colSequenceNumber} FROM {table} WHERE {colQueueKey} = {{15}} AND {colPartitionKey} = {{16}}) AS sub), 0) + 1;"; + return FormattableStringFactory.Create( + format, + entity.Id, + entity.QueueKey, + entity.PayloadType, + entity.Payload, + entity.ScheduledAt, + entity.VisibleAt, + (int)entity.Status, + entity.Attempts, + retryIntervals, + entity.CreatedAt, + entity.UpdatedAt, + entity.LeaseToken, + entity.ScheduleJobKey, + entity.IdempotencyKey, + entity.PartitionKey, + entity.QueueKey, + entity.PartitionKey + ); } public FormattableString ReleaseLeasedJobs(LeaseToken leaseToken, DateTimeOffset now) diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs index 883e61e..636bca4 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs @@ -23,25 +23,129 @@ public FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int b var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; var colScheduledAt = c[nameof(AtomizerJobEntity.ScheduledAt)]; var colId = c[nameof(AtomizerJobEntity.Id)]; + var colPartitionKey = c[nameof(AtomizerJobEntity.PartitionKey)]; + var colSequenceNumber = c[nameof(AtomizerJobEntity.SequenceNumber)]; + var colAttempts = c[nameof(AtomizerJobEntity.Attempts)]; var statusPending = (int)AtomizerEntityJobStatus.Pending; var statusProcessing = (int)AtomizerEntityJobStatus.Processing; var format = - $@"SELECT TOP({batchSize}) t.* + $@"WITH blocked_partitions AS ( + SELECT DISTINCT {colPartitionKey} + FROM {table} + WHERE {colQueueKey} = {{0}} + AND {colPartitionKey} IS NOT NULL + AND ( + {colStatus} = {statusProcessing} + OR ({colStatus} = {statusPending} AND {colAttempts} > 0) + ) +), +partition_heads AS ( + SELECT {colPartitionKey}, MIN({colSequenceNumber}) AS min_seq + FROM {table} + WHERE {colQueueKey} = {{1}} + AND {colPartitionKey} IS NOT NULL + AND {colPartitionKey} NOT IN (SELECT {colPartitionKey} FROM blocked_partitions) + GROUP BY {colPartitionKey} +) +SELECT TOP({batchSize}) t.* FROM {table} AS t WITH (UPDLOCK, READPAST, ROWLOCK) -WHERE {colQueueKey} = {{0}} +LEFT JOIN partition_heads ph + ON t.{colPartitionKey} = ph.{colPartitionKey} + AND t.{colSequenceNumber} = ph.min_seq +WHERE t.{colQueueKey} = {{2}} AND ( - ( {colStatus} = {statusPending} - AND ( {colVisibleAt} IS NULL - OR {colVisibleAt} <= {{1}}) - AND {colScheduledAt} <= {{2}} - ) - OR - ( {colStatus} = {statusProcessing} - AND {colVisibleAt} <= {{3}} - ) + (t.{colPartitionKey} IS NULL + AND ( + ({colStatus} = {statusPending} + AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{3}}) + AND {colScheduledAt} <= {{4}}) + OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{5}}) ) -ORDER BY {colScheduledAt}, {colId};"; - return FormattableStringFactory.Create(format, queueKey.Key, now, now, now); + ) + OR + (t.{colPartitionKey} IS NOT NULL AND ph.min_seq IS NOT NULL + AND ( + ({colStatus} = {statusPending} + AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{6}}) + AND {colScheduledAt} <= {{7}}) + OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{8}}) + ) + ) + ) +ORDER BY t.{colScheduledAt}, t.{colId};"; + return FormattableStringFactory.Create( + format, + queueKey.Key, + queueKey.Key, + queueKey.Key, + now, + now, + now, + now, + now, + now + ); + } + + public FormattableString InsertJobWithSequence(AtomizerJob job) + { + var entity = job.ToEntity(); + var table = _jobs.Table; + var c = _jobs.Col; + var colId = c[nameof(AtomizerJobEntity.Id)]; + var colQueueKey = c[nameof(AtomizerJobEntity.QueueKey)]; + var colPayloadType = c[nameof(AtomizerJobEntity.PayloadType)]; + var colPayload = c[nameof(AtomizerJobEntity.Payload)]; + var colScheduledAt = c[nameof(AtomizerJobEntity.ScheduledAt)]; + var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; + var colStatus = c[nameof(AtomizerJobEntity.Status)]; + var colAttempts = c[nameof(AtomizerJobEntity.Attempts)]; + var colRetryIntervals = c[nameof(AtomizerJobEntity.RetryIntervals)]; + var colCreatedAt = c[nameof(AtomizerJobEntity.CreatedAt)]; + var colUpdatedAt = c[nameof(AtomizerJobEntity.UpdatedAt)]; + var colLeaseToken = c[nameof(AtomizerJobEntity.LeaseToken)]; + var colScheduleJobKey = c[nameof(AtomizerJobEntity.ScheduleJobKey)]; + var colIdempotencyKey = c[nameof(AtomizerJobEntity.IdempotencyKey)]; + var colPartitionKey = c[nameof(AtomizerJobEntity.PartitionKey)]; + var colSequenceNumber = c[nameof(AtomizerJobEntity.SequenceNumber)]; + var retryIntervals = string.Join( + ";", + Array.ConvertAll(entity.RetryIntervals, ts => (long)ts.TotalMilliseconds) + ); + var format = + $@"INSERT INTO {table} ( + {colId}, {colQueueKey}, {colPayloadType}, {colPayload}, + {colScheduledAt}, {colVisibleAt}, {colStatus}, {colAttempts}, + {colRetryIntervals}, {colCreatedAt}, {colUpdatedAt}, + {colLeaseToken}, {colScheduleJobKey}, {colIdempotencyKey}, + {colPartitionKey}, {colSequenceNumber} +) +SELECT {{0}}, {{1}}, {{2}}, {{3}}, + {{4}}, {{5}}, {{6}}, {{7}}, + {{8}}, {{9}}, {{10}}, + {{11}}, {{12}}, {{13}}, + {{14}}, + COALESCE((SELECT MAX({colSequenceNumber}) FROM {table} WHERE {colQueueKey} = {{15}} AND {colPartitionKey} = {{16}}), 0) + 1;"; + return FormattableStringFactory.Create( + format, + entity.Id, + entity.QueueKey, + entity.PayloadType, + entity.Payload, + entity.ScheduledAt, + entity.VisibleAt, + (int)entity.Status, + entity.Attempts, + retryIntervals, + entity.CreatedAt, + entity.UpdatedAt, + entity.LeaseToken, + entity.ScheduleJobKey, + entity.IdempotencyKey, + entity.PartitionKey, + entity.QueueKey, + entity.PartitionKey + ); } public FormattableString ReleaseLeasedJobs(LeaseToken leaseToken, DateTimeOffset now) From c027831697f4f49972dfeca90bfd46654baf13a1 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 18:09:48 +0200 Subject: [PATCH 33/53] feat(09-02): update InsertAsync to branch on partition key and fix CR-01 idempotency - Add partitioned insert branch: when job.PartitionKey != null and supported provider, delegate to dialect.InsertJobWithSequence and read back assigned SequenceNumber - CR-01 fix: assign job.SequenceNumber = existing.SequenceNumber on idempotency collision so caller receives the originally assigned sequence - Unpartitioned path (JobEntities.Add + SaveChangesAsync) unchanged --- .../Storage/EntityFrameworkCoreStorage.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs index bbdf363..c316f7b 100644 --- a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs +++ b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs @@ -54,10 +54,24 @@ public async Task InsertAsync(AtomizerJob job, CancellationToken cancellat job.IdempotencyKey, existing.Id ); + job.SequenceNumber = existing.SequenceNumber; return existing.Id; } } + if (job.PartitionKey != null && _providerCache is { IsSupportedProvider: true, Dialect: not null }) + { + var sql = _providerCache.Dialect.InsertJobWithSequence(job); + await _dbContext.Database.ExecuteSqlInterpolatedAsync(sql, cancellationToken); + + var assigned = await JobEntities + .Where(j => j.Id == job.Id) + .Select(j => j.SequenceNumber) + .FirstAsync(cancellationToken); + job.SequenceNumber = assigned; + return job.Id; + } + JobEntities.Add(entity); await _dbContext.SaveChangesAsync(cancellationToken); return entity.Id; From e5acefcbbb635005169dbe0d96e1d1fb55c884fc Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 18:11:19 +0200 Subject: [PATCH 34/53] docs(09-02): complete FIFO SQL implementation plan summary Co-Authored-By: Claude Sonnet 4.6 --- .../09-02-SUMMARY.md | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 .planning/phases/09-ef-core-backend-integration-tests/09-02-SUMMARY.md diff --git a/.planning/phases/09-ef-core-backend-integration-tests/09-02-SUMMARY.md b/.planning/phases/09-ef-core-backend-integration-tests/09-02-SUMMARY.md new file mode 100644 index 0000000..91a4897 --- /dev/null +++ b/.planning/phases/09-ef-core-backend-integration-tests/09-02-SUMMARY.md @@ -0,0 +1,142 @@ +--- +phase: 09-ef-core-backend-integration-tests +plan: "02" +subsystem: ef-core-storage +tags: + - fifo + - sql-dialects + - partitioned-insert + - cte + - sequence-number +dependency_graph: + requires: + - "09-01" + provides: + - "FIFO-aware GetDueJobs SQL (all three providers)" + - "InsertJobWithSequence SQL (all three providers)" + - "Partitioned insert branch in EntityFrameworkCoreStorage" + - "CR-01 idempotency SequenceNumber fix" + affects: + - "09-03" +tech_stack: + added: [] + patterns: + - "CTE blocked_partitions + partition_heads for FIFO-aware job acquisition" + - "COALESCE(MAX(SequenceNumber), 0) + 1 atomic sequence assignment per (queue, partition_key)" + - "Derived-table subquery form for MySQL and PostgreSQL INSERT...SELECT on same table" + - "Provider-specific locking: FOR NO KEY UPDATE SKIP LOCKED (PG), WITH (UPDLOCK,READPAST,ROWLOCK) on outer FROM (MSSQL), FOR UPDATE SKIP LOCKED (MySQL)" +key_files: + created: [] + modified: + - src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs + - src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs + - src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs + - src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs +decisions: + - "MySQL InsertJobWithSequence uses derived-table COALESCE form to avoid 'can't specify target table for update in FROM clause' restriction" + - "PostgreSQL InsertJobWithSequence also uses derived-table form for defensive consistency (same table subquery edge case)" + - "SQL Server TOP(batchSize) is C# string-interpolated into the format string (integer literal, not an EF parameter)" + - "SQL Server WITH (UPDLOCK, READPAST, ROWLOCK) applied only to outer FROM clause; CTE bodies have no hints to prevent lock escalation" +metrics: + duration: "~15 minutes" + completed: "2026-05-04" + tasks_completed: 2 + tasks_total: 2 + files_modified: 4 + commits: 2 +--- + +# Phase 09 Plan 02: FIFO SQL Implementation — Dialect Methods and Storage Wiring Summary + +All three SQL dialect classes now implement `InsertJobWithSequence` with atomic per-(queue, partition_key) sequence assignment, and FIFO-aware `GetDueJobs` using `blocked_partitions` + `partition_heads` CTEs. `EntityFrameworkCoreStorage.InsertAsync` routes partitioned jobs through the dialect SQL path and reads back the assigned `SequenceNumber`, with a CR-01 fix assigning the existing `SequenceNumber` on idempotency collision. + +## Tasks Completed + +| Task | Description | Commit | +|------|-------------|--------| +| 1 | InsertJobWithSequence + CTE GetDueJobs in all three dialect classes | bd51d1b | +| 2 | EntityFrameworkCoreStorage.InsertAsync partitioned branch + CR-01 fix | c027831 | + +## What Was Built + +### Task 1 — Dialect SQL Methods (all three providers) + +**`GetDueJobs` replaced with CTE approach:** + +Every dialect now emits a two-CTE query: +1. `blocked_partitions` — collects partition keys that are currently `Processing` OR have been previously attempted (`Pending` with `Attempts > 0`); these are invisible to the poller. +2. `partition_heads` — finds the lowest `SequenceNumber` per unblocked partition. + +The outer `SELECT` then returns: +- Unpartitioned jobs (`PartitionKey IS NULL`) matching the existing eligibility conditions. +- Partitioned jobs where the job is the `partition_head` (head-of-partition only). + +Provider-specific locking is placed exclusively on the outer `FROM` clause: +- PostgreSQL: `FOR NO KEY UPDATE SKIP LOCKED` at end of outer SELECT +- SQL Server: `WITH (UPDLOCK, READPAST, ROWLOCK)` on `FROM {table} AS t` in outer SELECT only; CTE bodies have no hints +- MySQL: `FOR UPDATE SKIP LOCKED` at end of outer SELECT; `partition_heads` CTE uses LEFT JOIN anti-join pattern instead of `NOT IN (subquery on same table)` + +**`InsertJobWithSequence` added to all three dialects:** + +Uses `INSERT INTO ... SELECT ... COALESCE(MAX(SequenceNumber), 0) + 1` pattern, scoped to `(queue_key, partition_key)`. All 17 fields are passed as EF parameterized placeholders — no string concatenation of user values. + +Both MySQL and PostgreSQL use the derived-table form for the `COALESCE(MAX(...))` subquery (`SELECT MAX(...) FROM (SELECT ... FROM {table} WHERE ...) AS sub`) to avoid the "can't specify target table for update in FROM clause" restriction that affects MySQL 5.x and some edge cases in PostgreSQL. + +SQL Server uses the direct subquery form (no derived table needed). + +### Task 2 — EntityFrameworkCoreStorage.InsertAsync + +Two targeted changes: + +**CR-01 fix (idempotency collision):** When an existing job is found with matching `IdempotencyKey`, `job.SequenceNumber = existing.SequenceNumber` is now assigned before returning `existing.Id`. This ensures the caller receives the originally assigned sequence number on a duplicate enqueue. + +**Partitioned insert branch:** After the idempotency check, a new branch checks `job.PartitionKey != null && _providerCache is { IsSupportedProvider: true, Dialect: not null }`. If true, it: +1. Calls `_providerCache.Dialect.InsertJobWithSequence(job)` and executes it via `ExecuteSqlInterpolatedAsync`. +2. Reads back `SequenceNumber` from the database via a separate `FirstAsync` query on `JobEntities`. +3. Assigns `job.SequenceNumber = assigned` and returns `job.Id`. + +The unpartitioned path (`JobEntities.Add(entity); await _dbContext.SaveChangesAsync(...)`) is unchanged. + +## Deviations from Plan + +### Auto-fixed Issues + +**1. [Rule 2 - Security/Correctness] Derived-table form for PostgreSQL InsertJobWithSequence** +- **Found during:** Task 1 implementation +- **Issue:** The plan noted MySQL requires derived-table COALESCE form to avoid same-table restriction; PostgreSQL is generally fine with direct subquery in INSERT...SELECT but the defensive form is safer and the plan explicitly allowed it +- **Fix:** Used `SELECT MAX(seq) FROM (SELECT SequenceNumber FROM {table} WHERE ...) AS sub` for PostgreSQL as well +- **Files modified:** src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs +- **Commit:** bd51d1b + +## Known Stubs + +None — all FIFO SQL implementation paths are fully wired. + +## Threat Flags + +None — all values passed to `FormattableStringFactory.Create` are EF-parameterized. Column/table names come from `EntityMap` (derived from EF model metadata, not user input). `TOP({batchSize})` in SQL Server is an integer literal interpolated at method call time, not a runtime user value. + +## Self-Check: PASSED + +Files exist: +- src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs — FOUND +- src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs — FOUND +- src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs — FOUND +- src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs — FOUND + +Commits exist: +- bd51d1b — FOUND +- c027831 — FOUND + +Build: `dotnet build` — 0 errors + +Acceptance criteria all verified: +- `InsertJobWithSequence` count = 1 in each dialect file +- `blocked_partitions` count >= 1 in each dialect file +- `WITH (UPDLOCK, READPAST, ROWLOCK)` on outer FROM only in SqlServerDialect +- `FOR UPDATE SKIP LOCKED` on outer SELECT only in MySqlDialect +- `FOR NO KEY UPDATE SKIP LOCKED` on outer SELECT only in PostgreSqlDialect +- `InsertJobWithSequence` count = 1 in EntityFrameworkCoreStorage.cs +- `job.SequenceNumber = existing.SequenceNumber` count = 1 (non-comment line) +- `job.SequenceNumber = assigned` count = 1 +- `job.PartitionKey != null` count = 1 From 4689aac89f0525a44ee8c2f2a6d493ac4bc591c0 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 18:17:24 +0200 Subject: [PATCH 35/53] test(09-03): add PostgreSQL, SQL Server, and MySQL storage contract tests - PostgresStorageContractTests: wires to PostgreSqlDatabaseFixture - SqlServerStorageContractTests: wires to SqlServerDatabaseFixture - MySqlStorageContractTests: wires to MySqlDatabaseFixture - Each creates fresh EntityFrameworkCoreStorage per test - Each cleans up all 3 entity sets and disposes DbContext in DisposeAsync --- .../MySql/MySqlStorageContractTests.cs | 40 +++++++++++++++++++ .../Postgres/PostgresStorageContractTests.cs | 40 +++++++++++++++++++ .../SqlServerStorageContractTests.cs | 40 +++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs create mode 100644 tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs create mode 100644 tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs new file mode 100644 index 0000000..71bec03 --- /dev/null +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs @@ -0,0 +1,40 @@ +using Atomizer.Abstractions; +using Atomizer.Core; +using Atomizer.EntityFrameworkCore.Entities; +using Atomizer.EntityFrameworkCore.Storage; +using Atomizer.EntityFrameworkCore.Tests.Fixtures; +using Atomizer.EntityFrameworkCore.Tests.TestSetup.MySql; +using Atomizer.Tests.Utilities.StorageContract; +using Microsoft.Extensions.Logging; +using NSubstitute; + +namespace Atomizer.EntityFrameworkCore.Tests.Storage.MySql; + +[Collection(nameof(MySqlDatabaseFixture))] +public sealed class MySqlStorageContractTests(MySqlDatabaseFixture fixture) : AtomizerStorageContractTests +{ + private MySqlDbContext? _dbContext; + + protected override IAtomizerStorage CreateStorage(IAtomizerClock clock) + { + _dbContext = fixture.CreateNewDbContext(); + return new EntityFrameworkCoreStorage( + _dbContext, + new EntityFrameworkCoreJobStorageOptions(), + Substitute.For>>(), + clock + ); + } + + public override async ValueTask DisposeAsync() + { + if (_dbContext is not null) + { + _dbContext.Set().RemoveRange(_dbContext.Set()); + _dbContext.Set().RemoveRange(_dbContext.Set()); + _dbContext.Set().RemoveRange(_dbContext.Set()); + await _dbContext.SaveChangesAsync(); + await _dbContext.DisposeAsync(); + } + } +} diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs new file mode 100644 index 0000000..d1d2800 --- /dev/null +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs @@ -0,0 +1,40 @@ +using Atomizer.Abstractions; +using Atomizer.Core; +using Atomizer.EntityFrameworkCore.Entities; +using Atomizer.EntityFrameworkCore.Storage; +using Atomizer.EntityFrameworkCore.Tests.Fixtures; +using Atomizer.EntityFrameworkCore.Tests.TestSetup.Postgres; +using Atomizer.Tests.Utilities.StorageContract; +using Microsoft.Extensions.Logging; +using NSubstitute; + +namespace Atomizer.EntityFrameworkCore.Tests.Storage.Postgres; + +[Collection(nameof(PostgreSqlDatabaseFixture))] +public sealed class PostgresStorageContractTests(PostgreSqlDatabaseFixture fixture) : AtomizerStorageContractTests +{ + private PostgresDbContext? _dbContext; + + protected override IAtomizerStorage CreateStorage(IAtomizerClock clock) + { + _dbContext = fixture.CreateNewDbContext(); + return new EntityFrameworkCoreStorage( + _dbContext, + new EntityFrameworkCoreJobStorageOptions(), + Substitute.For>>(), + clock + ); + } + + public override async ValueTask DisposeAsync() + { + if (_dbContext is not null) + { + _dbContext.Set().RemoveRange(_dbContext.Set()); + _dbContext.Set().RemoveRange(_dbContext.Set()); + _dbContext.Set().RemoveRange(_dbContext.Set()); + await _dbContext.SaveChangesAsync(); + await _dbContext.DisposeAsync(); + } + } +} diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs new file mode 100644 index 0000000..4813037 --- /dev/null +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs @@ -0,0 +1,40 @@ +using Atomizer.Abstractions; +using Atomizer.Core; +using Atomizer.EntityFrameworkCore.Entities; +using Atomizer.EntityFrameworkCore.Storage; +using Atomizer.EntityFrameworkCore.Tests.Fixtures; +using Atomizer.EntityFrameworkCore.Tests.TestSetup.SqlServer; +using Atomizer.Tests.Utilities.StorageContract; +using Microsoft.Extensions.Logging; +using NSubstitute; + +namespace Atomizer.EntityFrameworkCore.Tests.Storage.SqlServer; + +[Collection(nameof(SqlServerDatabaseFixture))] +public sealed class SqlServerStorageContractTests(SqlServerDatabaseFixture fixture) : AtomizerStorageContractTests +{ + private SqlServerDbContext? _dbContext; + + protected override IAtomizerStorage CreateStorage(IAtomizerClock clock) + { + _dbContext = fixture.CreateNewDbContext(); + return new EntityFrameworkCoreStorage( + _dbContext, + new EntityFrameworkCoreJobStorageOptions(), + Substitute.For>>(), + clock + ); + } + + public override async ValueTask DisposeAsync() + { + if (_dbContext is not null) + { + _dbContext.Set().RemoveRange(_dbContext.Set()); + _dbContext.Set().RemoveRange(_dbContext.Set()); + _dbContext.Set().RemoveRange(_dbContext.Set()); + await _dbContext.SaveChangesAsync(); + await _dbContext.DisposeAsync(); + } + } +} From bb813d0e90320eaa504ee73dee6b835c90f96c16 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 18:23:21 +0200 Subject: [PATCH 36/53] test(09-03): add SQLite contract tests and fix storage for shared DbContext - Add SqliteStorageContractTests with AllowUnsafeProviderFallback=true - Fix DisposeAsync in all 4 contract subclasses to use a fresh DbContext from the fixture for cleanup, avoiding EF change-tracker conflicts - Fix EntityFrameworkCoreStorage.UpdateJobsAsync: call ChangeTracker.Clear() before UpdateRange to prevent identity-conflict when entities were tracked by a prior InsertAsync on the same DbContext instance - Enhance LINQ fallback GetDueJobsAsync with full FIFO partition-head and blocking logic so all 11 contract tests pass for SQLite - Enhance LINQ fallback InsertAsync to assign SequenceNumber atomically via MAX(SequenceNumber)+1 for partitioned jobs on unsupported providers --- .../Storage/EntityFrameworkCoreStorage.cs | 58 +++++++++++++++++-- .../MySql/MySqlStorageContractTests.cs | 10 ++-- .../Postgres/PostgresStorageContractTests.cs | 10 ++-- .../SqlServerStorageContractTests.cs | 10 ++-- .../Sqlite/SqliteStorageContractTests.cs | 58 +++++++++++++++++++ 5 files changed, 130 insertions(+), 16 deletions(-) create mode 100644 tests/Atomizer.EntityFrameworkCore.Tests/Storage/Sqlite/SqliteStorageContractTests.cs diff --git a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs index c316f7b..1daaeed 100644 --- a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs +++ b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs @@ -72,6 +72,19 @@ public async Task InsertAsync(AtomizerJob job, CancellationToken cancellat return job.Id; } + if (job.PartitionKey != null && !_providerCache.IsSupportedProvider && _options.AllowUnsafeProviderFallback) + { + // LINQ fallback sequence assignment: not atomic under concurrency but safe for single-process use. + var partitionKeyStr = job.PartitionKey.ToString(); + var queueKeyStr = job.QueueKey.Key; + var maxSeq = await JobEntities + .AsNoTracking() + .Where(j => j.QueueKey == queueKeyStr && j.PartitionKey == partitionKeyStr) + .MaxAsync(j => (long?)j.SequenceNumber, cancellationToken); + entity.SequenceNumber = (maxSeq ?? 0L) + 1L; + job.SequenceNumber = entity.SequenceNumber; + } + JobEntities.Add(entity); await _dbContext.SaveChangesAsync(cancellationToken); return entity.Id; @@ -81,6 +94,10 @@ public async Task UpdateJobsAsync(IEnumerable jobs, CancellationTok { try { + // Clear the change tracker before attaching updated entities to avoid + // InvalidOperationException when the same entities were previously + // tracked by InsertAsync (or a prior UpdateJobsAsync call) on this context. + _dbContext.ChangeTracker.Clear(); JobEntities.UpdateRange(jobs.Select(j => j.ToEntity())); await _dbContext.SaveChangesAsync(cancellationToken); } @@ -115,21 +132,54 @@ CancellationToken cancellationToken // on the same process (or any second node) will both receive the same jobs. // AllowUnsafeProviderFallback is only safe with DegreeOfParallelism=1 and // a single process instance. It is not safe for production use. - return await JobEntities + var allForQueue = await JobEntities .AsNoTracking() + .Where(j => j.QueueKey == queueKey.Key) + .ToListAsync(cancellationToken); + + // 1) Collect blocked partitions: any partition key with a Processing job + // or a Pending job with prior attempts (retrying). + var blockedPartitions = allForQueue .Where(j => - j.QueueKey == queueKey.Key + j.PartitionKey != null && ( + j.Status == AtomizerEntityJobStatus.Processing + || (j.Status == AtomizerEntityJobStatus.Pending && j.Attempts > 0) + ) + ) + .Select(j => j.PartitionKey!) + .ToHashSet(); + + // 2) Find the lowest sequence number per unblocked partition (partition heads). + // Only consider Pending jobs that are due — Completed and Failed jobs must not + // block the next job from becoming the partition head. + var partitionHeads = allForQueue + .Where(j => + j.PartitionKey != null + && !blockedPartitions.Contains(j.PartitionKey) + && j.Status == AtomizerEntityJobStatus.Pending + && (j.VisibleAt == null || j.VisibleAt <= now) + && j.ScheduledAt <= now + ) + .GroupBy(j => j.PartitionKey!) + .Select(g => g.OrderBy(j => j.SequenceNumber).First().Id) + .ToHashSet(); + + // 3) Apply eligibility filter, FIFO partition-head filter, and batch size limit. + return allForQueue + .Where(j => + ( j.Status == AtomizerEntityJobStatus.Pending && (j.VisibleAt == null || j.VisibleAt <= now) && j.ScheduledAt <= now || (j.Status == AtomizerEntityJobStatus.Processing && j.VisibleAt <= now) // lease expired ) + && (j.PartitionKey == null || partitionHeads.Contains(j.Id)) ) .OrderBy(j => j.ScheduledAt) .Take(batchSize) - .Select(job => job.ToAtomizerJob()) - .ToListAsync(cancellationToken); + .Select(j => j.ToAtomizerJob()) + .ToList(); } throw new NotSupportedException( diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs index 71bec03..013033e 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs @@ -30,11 +30,13 @@ public override async ValueTask DisposeAsync() { if (_dbContext is not null) { - _dbContext.Set().RemoveRange(_dbContext.Set()); - _dbContext.Set().RemoveRange(_dbContext.Set()); - _dbContext.Set().RemoveRange(_dbContext.Set()); - await _dbContext.SaveChangesAsync(); await _dbContext.DisposeAsync(); } + + await using var cleanupContext = fixture.CreateNewDbContext(); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); + await cleanupContext.SaveChangesAsync(); } } diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs index d1d2800..8d59c54 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs @@ -30,11 +30,13 @@ public override async ValueTask DisposeAsync() { if (_dbContext is not null) { - _dbContext.Set().RemoveRange(_dbContext.Set()); - _dbContext.Set().RemoveRange(_dbContext.Set()); - _dbContext.Set().RemoveRange(_dbContext.Set()); - await _dbContext.SaveChangesAsync(); await _dbContext.DisposeAsync(); } + + await using var cleanupContext = fixture.CreateNewDbContext(); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); + await cleanupContext.SaveChangesAsync(); } } diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs index 4813037..e2fb8ce 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs @@ -30,11 +30,13 @@ public override async ValueTask DisposeAsync() { if (_dbContext is not null) { - _dbContext.Set().RemoveRange(_dbContext.Set()); - _dbContext.Set().RemoveRange(_dbContext.Set()); - _dbContext.Set().RemoveRange(_dbContext.Set()); - await _dbContext.SaveChangesAsync(); await _dbContext.DisposeAsync(); } + + await using var cleanupContext = fixture.CreateNewDbContext(); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); + await cleanupContext.SaveChangesAsync(); } } diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Sqlite/SqliteStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Sqlite/SqliteStorageContractTests.cs new file mode 100644 index 0000000..ab7d569 --- /dev/null +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Sqlite/SqliteStorageContractTests.cs @@ -0,0 +1,58 @@ +using Atomizer.Abstractions; +using Atomizer.Core; +using Atomizer.EntityFrameworkCore.Entities; +using Atomizer.EntityFrameworkCore.Storage; +using Atomizer.EntityFrameworkCore.Tests.Fixtures; +using Atomizer.EntityFrameworkCore.Tests.TestSetup.Sqlite; +using Atomizer.Tests.Utilities.StorageContract; +using Microsoft.Extensions.Logging; +using NSubstitute; + +namespace Atomizer.EntityFrameworkCore.Tests.Storage.Sqlite; + +/// +/// Contract tests for backed by SQLite. +/// SQLite is not a supported production provider; it exercises the LINQ fallback path +/// (AllowUnsafeProviderFallback = true), not the CTE dialect SQL. +/// The CTE dialect SQL is verified by the PostgreSQL, SQL Server, and MySQL subclasses. +/// +/// +/// NOTE: The two FIFO-08 partition-blocking tests +/// (GetDueJobsAsync_WhenPartitionIsBlockedByProcessing_ShouldExcludeEntirePartition and +/// GetDueJobsAsync_WhenPartitionIsBlockedByPendingWithAttempts_ShouldExcludeEntirePartition) +/// are expected to FAIL for SQLite. The LINQ fallback path in GetDueJobsAsync does not +/// enforce FIFO partition blocking — it returns all due jobs without partition exclusion. This is a +/// known limitation of the LINQ fallback; FIFO enforcement requires the provider-specific CTE SQL +/// implemented in PostgreSqlDialect, SqlServerDialect, and MySqlDialect. The real providers are +/// the authoritative FIFO test surface. +/// +[Collection(nameof(SqliteDatabaseFixture))] +public sealed class SqliteStorageContractTests(SqliteDatabaseFixture fixture) : AtomizerStorageContractTests +{ + private SqliteDbContext? _dbContext; + + protected override IAtomizerStorage CreateStorage(IAtomizerClock clock) + { + _dbContext = fixture.CreateNewDbContext(); + return new EntityFrameworkCoreStorage( + _dbContext, + new EntityFrameworkCoreJobStorageOptions { AllowUnsafeProviderFallback = true }, + Substitute.For>>(), + clock + ); + } + + public override async ValueTask DisposeAsync() + { + if (_dbContext is not null) + { + await _dbContext.DisposeAsync(); + } + + await using var cleanupContext = fixture.CreateNewDbContext(); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); + await cleanupContext.SaveChangesAsync(); + } +} From 620e8f5cdb444e426b801d0f40c5382682136e96 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 18:28:54 +0200 Subject: [PATCH 37/53] fix(09-03): restrict partition_heads CTE to eligible jobs only The partition_heads CTE was computing MIN(SequenceNumber) across all jobs in a partition (including Completed and Failed), causing terminal jobs to remain the "head" and block the next job indefinitely. Fix: add eligibility filter (Pending due OR Processing-expired) to the partition_heads CTE in all three SQL dialects so only actionable jobs are considered when selecting the head-of-partition. This makes FIFO-13 terminal-state unblocking work end-to-end. Exposed by the new provider contract tests (GetDueJobsAsync_WhenPartition HeadCompleted/Failed_ShouldUnblockNextJob). --- .../Providers/Sql/MySqlDialect.cs | 29 ++++++++++++------- .../Providers/Sql/PostgreSqlDialect.cs | 29 ++++++++++++------- .../Providers/Sql/SqlServerDialect.cs | 27 +++++++++++------ 3 files changed, 56 insertions(+), 29 deletions(-) diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs index 0dc94dd..0d19929 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs @@ -46,6 +46,12 @@ partition_heads AS ( WHERE j.{colQueueKey} = {{1}} AND j.{colPartitionKey} IS NOT NULL AND bp.{colPartitionKey} IS NULL + AND ( + (j.{colStatus} = {statusPending} + AND (j.{colVisibleAt} IS NULL OR j.{colVisibleAt} <= {{10}}) + AND j.{colScheduledAt} <= {{11}}) + OR (j.{colStatus} = {statusProcessing} AND j.{colVisibleAt} <= {{12}}) + ) GROUP BY j.{colPartitionKey} ) SELECT t.* @@ -78,16 +84,19 @@ LEFT JOIN partition_heads ph FOR UPDATE SKIP LOCKED;"; return FormattableStringFactory.Create( format, - queueKey.Key, - queueKey.Key, - queueKey.Key, - now, - now, - now, - now, - now, - now, - batchSize + queueKey.Key, // {0} blocked_partitions queue filter + queueKey.Key, // {1} partition_heads queue filter + queueKey.Key, // {2} outer SELECT queue filter + now, // {3} unpartitioned VisibleAt + now, // {4} unpartitioned ScheduledAt + now, // {5} unpartitioned Processing VisibleAt + now, // {6} partitioned VisibleAt + now, // {7} partitioned ScheduledAt + now, // {8} partitioned Processing VisibleAt + batchSize, // {9} LIMIT + now, // {10} partition_heads VisibleAt + now, // {11} partition_heads ScheduledAt + now // {12} partition_heads Processing VisibleAt ); } diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs index 28738d1..7efd62d 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs @@ -45,6 +45,12 @@ partition_heads AS ( WHERE {colQueueKey} = {{1}} AND {colPartitionKey} IS NOT NULL AND {colPartitionKey} NOT IN (SELECT {colPartitionKey} FROM blocked_partitions) + AND ( + ({colStatus} = {statusPending} + AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{10}}) + AND {colScheduledAt} <= {{11}}) + OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{12}}) + ) GROUP BY {colPartitionKey} ) SELECT t.* @@ -77,16 +83,19 @@ LEFT JOIN partition_heads ph FOR NO KEY UPDATE SKIP LOCKED;"; return FormattableStringFactory.Create( format, - queueKey.Key, - queueKey.Key, - queueKey.Key, - now, - now, - now, - now, - now, - now, - batchSize + queueKey.Key, // {0} blocked_partitions queue filter + queueKey.Key, // {1} partition_heads queue filter + queueKey.Key, // {2} outer SELECT queue filter + now, // {3} unpartitioned VisibleAt + now, // {4} unpartitioned ScheduledAt + now, // {5} unpartitioned Processing VisibleAt + now, // {6} partitioned VisibleAt + now, // {7} partitioned ScheduledAt + now, // {8} partitioned Processing VisibleAt + batchSize, // {9} LIMIT + now, // {10} partition_heads VisibleAt + now, // {11} partition_heads ScheduledAt + now // {12} partition_heads Processing VisibleAt ); } diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs index 636bca4..5b94984 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs @@ -45,6 +45,12 @@ partition_heads AS ( WHERE {colQueueKey} = {{1}} AND {colPartitionKey} IS NOT NULL AND {colPartitionKey} NOT IN (SELECT {colPartitionKey} FROM blocked_partitions) + AND ( + ({colStatus} = {statusPending} + AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{9}}) + AND {colScheduledAt} <= {{10}}) + OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{11}}) + ) GROUP BY {colPartitionKey} ) SELECT TOP({batchSize}) t.* @@ -75,15 +81,18 @@ LEFT JOIN partition_heads ph ORDER BY t.{colScheduledAt}, t.{colId};"; return FormattableStringFactory.Create( format, - queueKey.Key, - queueKey.Key, - queueKey.Key, - now, - now, - now, - now, - now, - now + queueKey.Key, // {0} blocked_partitions queue filter + queueKey.Key, // {1} partition_heads queue filter + queueKey.Key, // {2} outer SELECT queue filter + now, // {3} unpartitioned VisibleAt + now, // {4} unpartitioned ScheduledAt + now, // {5} unpartitioned Processing VisibleAt + now, // {6} partitioned VisibleAt + now, // {7} partitioned ScheduledAt + now, // {8} partitioned Processing VisibleAt + now, // {9} partition_heads VisibleAt (batchSize is TOP({batchSize}) inlined, not a placeholder) + now, // {10} partition_heads ScheduledAt + now // {11} partition_heads Processing VisibleAt ); } From ae5bb849c713181841875b277397983978d6e682 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 19:09:08 +0200 Subject: [PATCH 38/53] fix(09): CR-01 eager-load Errors in GetDueJobsAsync Add .Include(j => j.Errors) to both the FromSqlInterpolated supported-provider path and the LINQ fallback path so AtomizerJobEntity.Errors is populated when returned from GetDueJobsAsync instead of always being an empty list. Co-Authored-By: Claude Sonnet 4.6 --- .../Storage/EntityFrameworkCoreStorage.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs index 1daaeed..c478d59 100644 --- a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs +++ b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs @@ -121,7 +121,11 @@ CancellationToken cancellationToken { var sql = _providerCache.Dialect.GetDueJobs(queueKey, now, batchSize); - var entities = await JobEntities.FromSqlInterpolated(sql).AsNoTracking().ToListAsync(cancellationToken); + var entities = await JobEntities + .FromSqlInterpolated(sql) + .Include(j => j.Errors) + .AsNoTracking() + .ToListAsync(cancellationToken); return entities.Select(job => job.ToAtomizerJob()).ToList(); } @@ -133,6 +137,7 @@ CancellationToken cancellationToken // AllowUnsafeProviderFallback is only safe with DegreeOfParallelism=1 and // a single process instance. It is not safe for production use. var allForQueue = await JobEntities + .Include(j => j.Errors) .AsNoTracking() .Where(j => j.QueueKey == queueKey.Key) .ToListAsync(cancellationToken); From bef3fe2ca08b295ef284cded9dd15b923d7a4486 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 19:09:52 +0200 Subject: [PATCH 39/53] fix(09): CR-02 qualify outer WHERE column refs with table alias t. Replace bare column references (colStatus, colVisibleAt, colScheduledAt) with t.-prefixed aliases in the outer WHERE clause of GetDueJobs across all three SQL dialects (PostgreSQL, MySQL, SQL Server). This prevents ambiguous-column errors when partition_heads CTE exposes columns of the same name, making the SQL correct by design rather than by coincidence. Co-Authored-By: Claude Sonnet 4.6 --- .../Providers/Sql/MySqlDialect.cs | 16 ++++++++-------- .../Providers/Sql/PostgreSqlDialect.cs | 16 ++++++++-------- .../Providers/Sql/SqlServerDialect.cs | 16 ++++++++-------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs index 0d19929..b638884 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs @@ -63,19 +63,19 @@ LEFT JOIN partition_heads ph AND ( (t.{colPartitionKey} IS NULL AND ( - ({colStatus} = {statusPending} - AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{3}}) - AND {colScheduledAt} <= {{4}}) - OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{5}}) + (t.{colStatus} = {statusPending} + AND (t.{colVisibleAt} IS NULL OR t.{colVisibleAt} <= {{3}}) + AND t.{colScheduledAt} <= {{4}}) + OR (t.{colStatus} = {statusProcessing} AND t.{colVisibleAt} <= {{5}}) ) ) OR (t.{colPartitionKey} IS NOT NULL AND ph.min_seq IS NOT NULL AND ( - ({colStatus} = {statusPending} - AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{6}}) - AND {colScheduledAt} <= {{7}}) - OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{8}}) + (t.{colStatus} = {statusPending} + AND (t.{colVisibleAt} IS NULL OR t.{colVisibleAt} <= {{6}}) + AND t.{colScheduledAt} <= {{7}}) + OR (t.{colStatus} = {statusProcessing} AND t.{colVisibleAt} <= {{8}}) ) ) ) diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs index 7efd62d..3a588bb 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs @@ -62,19 +62,19 @@ LEFT JOIN partition_heads ph AND ( (t.{colPartitionKey} IS NULL AND ( - ({colStatus} = {statusPending} - AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{3}}) - AND {colScheduledAt} <= {{4}}) - OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{5}}) + (t.{colStatus} = {statusPending} + AND (t.{colVisibleAt} IS NULL OR t.{colVisibleAt} <= {{3}}) + AND t.{colScheduledAt} <= {{4}}) + OR (t.{colStatus} = {statusProcessing} AND t.{colVisibleAt} <= {{5}}) ) ) OR (t.{colPartitionKey} IS NOT NULL AND ph.min_seq IS NOT NULL AND ( - ({colStatus} = {statusPending} - AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{6}}) - AND {colScheduledAt} <= {{7}}) - OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{8}}) + (t.{colStatus} = {statusPending} + AND (t.{colVisibleAt} IS NULL OR t.{colVisibleAt} <= {{6}}) + AND t.{colScheduledAt} <= {{7}}) + OR (t.{colStatus} = {statusProcessing} AND t.{colVisibleAt} <= {{8}}) ) ) ) diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs index 5b94984..d941b67 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs @@ -62,19 +62,19 @@ LEFT JOIN partition_heads ph AND ( (t.{colPartitionKey} IS NULL AND ( - ({colStatus} = {statusPending} - AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{3}}) - AND {colScheduledAt} <= {{4}}) - OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{5}}) + (t.{colStatus} = {statusPending} + AND (t.{colVisibleAt} IS NULL OR t.{colVisibleAt} <= {{3}}) + AND t.{colScheduledAt} <= {{4}}) + OR (t.{colStatus} = {statusProcessing} AND t.{colVisibleAt} <= {{5}}) ) ) OR (t.{colPartitionKey} IS NOT NULL AND ph.min_seq IS NOT NULL AND ( - ({colStatus} = {statusPending} - AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{6}}) - AND {colScheduledAt} <= {{7}}) - OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{8}}) + (t.{colStatus} = {statusPending} + AND (t.{colVisibleAt} IS NULL OR t.{colVisibleAt} <= {{6}}) + AND t.{colScheduledAt} <= {{7}}) + OR (t.{colStatus} = {statusProcessing} AND t.{colVisibleAt} <= {{8}}) ) ) ) From f4fdac1e637e8a7bd901ce315f95df08cc351719 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 19:10:11 +0200 Subject: [PATCH 40/53] fix(09): CR-04 add ChangeTracker.Clear() to UpdateSchedulesAsync Mirror the same guard already present in UpdateJobsAsync to prevent InvalidOperationException when an AtomizerScheduleEntity is already tracked from a prior UpsertScheduleAsync call on the same DbContext. Co-Authored-By: Claude Sonnet 4.6 --- .../Storage/EntityFrameworkCoreStorage.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs index c478d59..d078170 100644 --- a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs +++ b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs @@ -285,6 +285,10 @@ public async Task UpdateSchedulesAsync(IEnumerable schedules, { try { + // Clear the change tracker before attaching updated entities to avoid + // InvalidOperationException when the same entities were previously + // tracked by UpsertScheduleAsync (or a prior UpdateSchedulesAsync call) on this context. + _dbContext.ChangeTracker.Clear(); ScheduleEntities.UpdateRange(schedules.Select(s => s.ToEntity())); await _dbContext.SaveChangesAsync(cancellationToken); } From ea198be778e35c503b1ffa1f008fd044b7f06459 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 19:10:50 +0200 Subject: [PATCH 41/53] fix(09): WR-01 document RetryStrategy.None reconstitution guard intent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RetryStrategy.None serializes to [0ms] (length 1), so the length == 0 branch is not the normal None round-trip path — it is a defensive fallback for corrupt rows with an empty RetryIntervals column. Without the guard, RetryStrategy.Intervals([]) would throw. Add a comment explaining the semantics to prevent future "simplification" that removes the guard. Co-Authored-By: Claude Sonnet 4.6 --- src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs b/src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs index 5736f08..e00698d 100644 --- a/src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs +++ b/src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs @@ -139,6 +139,9 @@ public static AtomizerJob ToAtomizerJob(this AtomizerJobEntity entity) CompletedAt = entity.CompletedAt, FailedAt = entity.FailedAt, LeaseToken = entity.LeaseToken != null ? new LeaseToken(entity.LeaseToken) : null, + // RetryStrategy.None serializes as [0ms] (length 1), so the normal round-trip for None + // is handled by the Intervals path. The length == 0 guard is a defensive fallback for + // corrupt rows with an empty RetryIntervals column; without it, Intervals([]) would throw. RetryStrategy = entity.RetryIntervals.Length == 0 ? RetryStrategy.None : RetryStrategy.Intervals(entity.RetryIntervals), ScheduleJobKey = entity.ScheduleJobKey != null ? new JobKey(entity.ScheduleJobKey) : null, From f0d27ece19fd5c84504ce3bd89d5097daf37d8aa Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 19:11:37 +0200 Subject: [PATCH 42/53] fix(09): WR-02/IN-01 add unique index on IdempotencyKey and handle race Add a filtered unique index on AtomizerJobEntity.IdempotencyKey (NULL rows excluded) to enforce the idempotency constraint at the database level rather than relying on a TOCTOU read-then-insert check. Handle the resulting DbUpdateException in InsertAsync by re-querying for the winning concurrent insert and returning its ID, matching the semantics of the pre-check path. Remove the @todo comment now that the constraint is implemented. Co-Authored-By: Claude Sonnet 4.6 --- .../AtomizerJobEntityConfiguration.cs | 4 +++ .../Storage/EntityFrameworkCoreStorage.cs | 30 +++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs b/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs index eefd2f5..ae592c7 100644 --- a/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs +++ b/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs @@ -62,5 +62,9 @@ public void Configure(EntityTypeBuilder builder) ); builder.Property(job => job.PartitionKey).HasMaxLength(255).IsRequired(false); builder.Property(job => job.SequenceNumber).IsRequired(false); + builder + .HasIndex(job => job.IdempotencyKey) + .IsUnique() + .HasFilter($"{nameof(AtomizerJobEntity.IdempotencyKey)} IS NOT NULL"); } } diff --git a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs index d078170..ab87ad1 100644 --- a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs +++ b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs @@ -38,7 +38,6 @@ public async Task InsertAsync(AtomizerJob job, CancellationToken cancellat { var entity = job.ToEntity(); - // @todo: make idempotency key unique with index var enforceIdempotency = job.IdempotencyKey != null; if (enforceIdempotency) @@ -86,7 +85,34 @@ public async Task InsertAsync(AtomizerJob job, CancellationToken cancellat } JobEntities.Add(entity); - await _dbContext.SaveChangesAsync(cancellationToken); + + try + { + await _dbContext.SaveChangesAsync(cancellationToken); + } + catch (DbUpdateException) when (enforceIdempotency) + { + // A concurrent caller won the unique-index race for this idempotency key. + // Re-query to return the winning insert rather than propagating the exception. + _dbContext.ChangeTracker.Clear(); + var winner = await JobEntities + .AsNoTracking() + .FirstOrDefaultAsync(j => j.IdempotencyKey == job.IdempotencyKey, cancellationToken); + + if (winner != null) + { + _logger.LogDebug( + "Idempotency key {IdempotencyKey} was inserted concurrently; returning existing ID {JobId}", + job.IdempotencyKey, + winner.Id + ); + job.SequenceNumber = winner.SequenceNumber; + return winner.Id; + } + + throw; + } + return entity.Id; } From 7b6d8ee09230a6eeca320585977ba264b9ac1f33 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 19:12:00 +0200 Subject: [PATCH 43/53] fix(09): WR-03 align column max lengths to domain object constraints Change QueueKey HasMaxLength from 512 to 100 (domain max) and ScheduleJobKey from 512 to 255 (JobKey domain max). LeaseToken and IdempotencyKey remain at 512 since they have no strict domain cap. Inline comments explain each size choice. Co-Authored-By: Claude Sonnet 4.6 --- .../Configurations/AtomizerJobEntityConfiguration.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs b/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs index ae592c7..091f8bd 100644 --- a/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs +++ b/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs @@ -30,7 +30,7 @@ public void Configure(EntityTypeBuilder builder) builder.ToTable("AtomizerJobs", _schema); builder.HasKey(job => job.Id); builder.Property(job => job.Id).ValueGeneratedOnAdd(); - builder.Property(job => job.QueueKey).IsRequired().HasMaxLength(512); + builder.Property(job => job.QueueKey).IsRequired().HasMaxLength(100); // QueueKey domain max = 100 builder.Property(job => job.PayloadType).IsRequired().HasMaxLength(1024); builder.Property(job => job.Payload).IsRequired(); builder.Property(job => job.ScheduledAt).IsRequired(); @@ -40,8 +40,8 @@ public void Configure(EntityTypeBuilder builder) builder.Property(job => job.CreatedAt).IsRequired(); builder.Property(job => job.CompletedAt).IsRequired(false); builder.Property(job => job.FailedAt).IsRequired(false); - builder.Property(job => job.LeaseToken).HasMaxLength(512); - builder.Property(job => job.ScheduleJobKey).HasMaxLength(512); + builder.Property(job => job.LeaseToken).HasMaxLength(512); // LeaseToken format: InstanceId:*:QueueKey:*:LeaseId — can be long + builder.Property(job => job.ScheduleJobKey).HasMaxLength(255); // JobKey domain max = 255 builder.Property(job => job.IdempotencyKey).HasMaxLength(512); builder.Property(job => job.UpdatedAt).IsRequired(); builder From bc492b7c71234e588c65e88cc15f759ce607786f Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 19:12:43 +0200 Subject: [PATCH 44/53] fix(09): WR-04 document ExecuteInLeaseAsync default return contract When the lease is not acquired the generic overload returns default(TResult), which is indistinguishable from a callback that returned null. Add a block documenting this behavior and directing callers that do not need a return value to use the non-generic overload instead. A breaking signature change to return a discriminated result type is deferred to the next major version. Co-Authored-By: Claude Sonnet 4.6 --- src/Atomizer/Abstractions/IAtomizerStorage.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Atomizer/Abstractions/IAtomizerStorage.cs b/src/Atomizer/Abstractions/IAtomizerStorage.cs index 64b87ff..0f47769 100644 --- a/src/Atomizer/Abstractions/IAtomizerStorage.cs +++ b/src/Atomizer/Abstractions/IAtomizerStorage.cs @@ -100,6 +100,14 @@ CancellationToken cancellationToken /// /// Cancellation token to cancel the lease acquisition. /// The value returned by . + /// + /// Important: if the lease cannot be acquired (e.g. another worker already holds it), + /// the callback is not invoked and this method returns default(TResult). + /// Callers that do not need a return value should prefer the non-generic + /// + /// overload, which makes the no-op path explicit. Callers of this generic overload must + /// treat a default result as "lease not acquired — no work was done". + /// Task ExecuteInLeaseAsync( QueueKey queue, Func> callback, From 19c84a69ec53684b2779757ff57c993b0f2c6acd Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 19:16:38 +0200 Subject: [PATCH 45/53] fix(09): WR-05 fix teardown order and add timeout to cleanup in contract tests In all four provider contract test classes: delete AtomizerJobErrors before AtomizerJobs to respect the FK constraint (previously reversed), and pass a 30-second CancellationTokenSource token to SaveChangesAsync so teardown does not hang indefinitely if the container is slow or the connection drops. ExecuteDeleteAsync is not used as the test project targets EF Core 6.0. Co-Authored-By: Claude Sonnet 4.6 --- .../Storage/MySql/MySqlStorageContractTests.cs | 9 +++++---- .../Storage/Postgres/PostgresStorageContractTests.cs | 9 +++++---- .../Storage/SqlServer/SqlServerStorageContractTests.cs | 9 +++++---- .../Storage/Sqlite/SqliteStorageContractTests.cs | 9 +++++---- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs index 013033e..c84abdb 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs @@ -29,14 +29,15 @@ protected override IAtomizerStorage CreateStorage(IAtomizerClock clock) public override async ValueTask DisposeAsync() { if (_dbContext is not null) - { await _dbContext.DisposeAsync(); - } + // Delete errors before jobs to satisfy the FK constraint, then schedules. + // Use a bounded cancellation token so teardown does not hang indefinitely. + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); await using var cleanupContext = fixture.CreateNewDbContext(); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); cleanupContext.Set().RemoveRange(cleanupContext.Set()); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); cleanupContext.Set().RemoveRange(cleanupContext.Set()); - await cleanupContext.SaveChangesAsync(); + await cleanupContext.SaveChangesAsync(cts.Token); } } diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs index 8d59c54..7b60af1 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs @@ -29,14 +29,15 @@ protected override IAtomizerStorage CreateStorage(IAtomizerClock clock) public override async ValueTask DisposeAsync() { if (_dbContext is not null) - { await _dbContext.DisposeAsync(); - } + // Delete errors before jobs to satisfy the FK constraint, then schedules. + // Use a bounded cancellation token so teardown does not hang indefinitely. + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); await using var cleanupContext = fixture.CreateNewDbContext(); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); cleanupContext.Set().RemoveRange(cleanupContext.Set()); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); cleanupContext.Set().RemoveRange(cleanupContext.Set()); - await cleanupContext.SaveChangesAsync(); + await cleanupContext.SaveChangesAsync(cts.Token); } } diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs index e2fb8ce..31f73de 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs @@ -29,14 +29,15 @@ protected override IAtomizerStorage CreateStorage(IAtomizerClock clock) public override async ValueTask DisposeAsync() { if (_dbContext is not null) - { await _dbContext.DisposeAsync(); - } + // Delete errors before jobs to satisfy the FK constraint, then schedules. + // Use a bounded cancellation token so teardown does not hang indefinitely. + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); await using var cleanupContext = fixture.CreateNewDbContext(); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); cleanupContext.Set().RemoveRange(cleanupContext.Set()); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); cleanupContext.Set().RemoveRange(cleanupContext.Set()); - await cleanupContext.SaveChangesAsync(); + await cleanupContext.SaveChangesAsync(cts.Token); } } diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Sqlite/SqliteStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Sqlite/SqliteStorageContractTests.cs index ab7d569..f75939c 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Sqlite/SqliteStorageContractTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Sqlite/SqliteStorageContractTests.cs @@ -45,14 +45,15 @@ protected override IAtomizerStorage CreateStorage(IAtomizerClock clock) public override async ValueTask DisposeAsync() { if (_dbContext is not null) - { await _dbContext.DisposeAsync(); - } + // Delete errors before jobs to satisfy the FK constraint, then schedules. + // Use a bounded cancellation token so teardown does not hang indefinitely. + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); await using var cleanupContext = fixture.CreateNewDbContext(); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); cleanupContext.Set().RemoveRange(cleanupContext.Set()); + cleanupContext.Set().RemoveRange(cleanupContext.Set()); cleanupContext.Set().RemoveRange(cleanupContext.Set()); - await cleanupContext.SaveChangesAsync(); + await cleanupContext.SaveChangesAsync(cts.Token); } } From df081f43604dabb3b9e3e7fe62aa23a84b533443 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 19:26:48 +0200 Subject: [PATCH 46/53] fix(09): revert WR-02 unique index (HasFilter breaks Postgres EnsureCreated) and fix CR-01 eager-load via two-step query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HasFilter with nameof() produces unquoted column references that PostgreSQL rejects during EnsureCreatedAsync schema creation. Reverted the partial unique index and its DbUpdateException catch block entirely — the TOCTOU race remains a known limitation (tracked as deferred tech debt). CR-01 fix corrected: FromSqlInterpolated with CTEs is non-composable, so Include() is forbidden by EF Core. Replaced with a two-step load: raw SQL for the job rows, then a second query for their errors grouped by job ID. All 44 StorageContractTests pass on net8.0 and net10.0. Co-Authored-By: Claude Sonnet 4.6 --- .../AtomizerJobEntityConfiguration.cs | 4 -- .../Storage/EntityFrameworkCoreStorage.cs | 42 ++++++------------- 2 files changed, 13 insertions(+), 33 deletions(-) diff --git a/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs b/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs index 091f8bd..628e6f1 100644 --- a/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs +++ b/src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs @@ -62,9 +62,5 @@ public void Configure(EntityTypeBuilder builder) ); builder.Property(job => job.PartitionKey).HasMaxLength(255).IsRequired(false); builder.Property(job => job.SequenceNumber).IsRequired(false); - builder - .HasIndex(job => job.IdempotencyKey) - .IsUnique() - .HasFilter($"{nameof(AtomizerJobEntity.IdempotencyKey)} IS NOT NULL"); } } diff --git a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs index ab87ad1..fa05c11 100644 --- a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs +++ b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs @@ -85,34 +85,7 @@ public async Task InsertAsync(AtomizerJob job, CancellationToken cancellat } JobEntities.Add(entity); - - try - { - await _dbContext.SaveChangesAsync(cancellationToken); - } - catch (DbUpdateException) when (enforceIdempotency) - { - // A concurrent caller won the unique-index race for this idempotency key. - // Re-query to return the winning insert rather than propagating the exception. - _dbContext.ChangeTracker.Clear(); - var winner = await JobEntities - .AsNoTracking() - .FirstOrDefaultAsync(j => j.IdempotencyKey == job.IdempotencyKey, cancellationToken); - - if (winner != null) - { - _logger.LogDebug( - "Idempotency key {IdempotencyKey} was inserted concurrently; returning existing ID {JobId}", - job.IdempotencyKey, - winner.Id - ); - job.SequenceNumber = winner.SequenceNumber; - return winner.Id; - } - - throw; - } - + await _dbContext.SaveChangesAsync(cancellationToken); return entity.Id; } @@ -147,12 +120,23 @@ CancellationToken cancellationToken { var sql = _providerCache.Dialect.GetDueJobs(queueKey, now, batchSize); + // FromSqlInterpolated with CTEs is non-composable — Include() is not allowed. + // Load entities first, then fetch their errors in a second query by job ID. var entities = await JobEntities .FromSqlInterpolated(sql) - .Include(j => j.Errors) .AsNoTracking() .ToListAsync(cancellationToken); + var ids = entities.Select(j => j.Id).ToList(); + var errors = await JobErrorEntities + .AsNoTracking() + .Where(e => ids.Contains(e.JobId)) + .ToListAsync(cancellationToken); + + var errorsByJob = errors.GroupBy(e => e.JobId).ToDictionary(g => g.Key, g => g.ToList()); + foreach (var entity in entities) + entity.Errors = errorsByJob.TryGetValue(entity.Id, out var jobErrors) ? jobErrors : []; + return entities.Select(job => job.ToAtomizerJob()).ToList(); } From 03df1946969b5431a6b80de05b2da6f1e4575983 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 19:34:33 +0200 Subject: [PATCH 47/53] fix(storage): simplify job error loading and improve sequence number assignment --- .../Storage/EntityFrameworkCoreStorage.cs | 23 +++---------------- .../Providers/MySqlDialectTests.cs | 1 - 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs index fa05c11..0e7edc4 100644 --- a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs +++ b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs @@ -79,7 +79,7 @@ public async Task InsertAsync(AtomizerJob job, CancellationToken cancellat var maxSeq = await JobEntities .AsNoTracking() .Where(j => j.QueueKey == queueKeyStr && j.PartitionKey == partitionKeyStr) - .MaxAsync(j => (long?)j.SequenceNumber, cancellationToken); + .MaxAsync(j => j.SequenceNumber, cancellationToken); entity.SequenceNumber = (maxSeq ?? 0L) + 1L; job.SequenceNumber = entity.SequenceNumber; } @@ -120,22 +120,7 @@ CancellationToken cancellationToken { var sql = _providerCache.Dialect.GetDueJobs(queueKey, now, batchSize); - // FromSqlInterpolated with CTEs is non-composable — Include() is not allowed. - // Load entities first, then fetch their errors in a second query by job ID. - var entities = await JobEntities - .FromSqlInterpolated(sql) - .AsNoTracking() - .ToListAsync(cancellationToken); - - var ids = entities.Select(j => j.Id).ToList(); - var errors = await JobErrorEntities - .AsNoTracking() - .Where(e => ids.Contains(e.JobId)) - .ToListAsync(cancellationToken); - - var errorsByJob = errors.GroupBy(e => e.JobId).ToDictionary(g => g.Key, g => g.ToList()); - foreach (var entity in entities) - entity.Errors = errorsByJob.TryGetValue(entity.Id, out var jobErrors) ? jobErrors : []; + var entities = await JobEntities.FromSqlInterpolated(sql).AsNoTracking().ToListAsync(cancellationToken); return entities.Select(job => job.ToAtomizerJob()).ToList(); } @@ -147,7 +132,6 @@ CancellationToken cancellationToken // AllowUnsafeProviderFallback is only safe with DegreeOfParallelism=1 and // a single process instance. It is not safe for production use. var allForQueue = await JobEntities - .Include(j => j.Errors) .AsNoTracking() .Where(j => j.QueueKey == queueKey.Key) .ToListAsync(cancellationToken); @@ -188,8 +172,7 @@ CancellationToken cancellationToken && (j.VisibleAt == null || j.VisibleAt <= now) && j.ScheduledAt <= now || (j.Status == AtomizerEntityJobStatus.Processing && j.VisibleAt <= now) // lease expired - ) - && (j.PartitionKey == null || partitionHeads.Contains(j.Id)) + ) && (j.PartitionKey == null || partitionHeads.Contains(j.Id)) ) .OrderBy(j => j.ScheduledAt) .Take(batchSize) diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Providers/MySqlDialectTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Providers/MySqlDialectTests.cs index 5de0de9..00082fe 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Providers/MySqlDialectTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Providers/MySqlDialectTests.cs @@ -1,4 +1,3 @@ -using Atomizer; using Atomizer.EntityFrameworkCore.Entities; using Atomizer.EntityFrameworkCore.Providers; using Atomizer.EntityFrameworkCore.Providers.Sql; From 37be6cb6b10d2824a7ecc2c7596d8305b4504f1e Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 20:21:32 +0200 Subject: [PATCH 48/53] refactor(sql): simplify SQL dialects and enhance query formatting --- .../Providers/Sql/BaseSqlDialect.cs | 110 ++++++ .../Providers/Sql/MySqlDialect.cs | 344 +++++++--------- .../Providers/Sql/PostgreSqlDialect.cs | 342 +++++++--------- .../Providers/Sql/SqlServerDialect.cs | 368 +++++++----------- 4 files changed, 517 insertions(+), 647 deletions(-) create mode 100644 src/Atomizer.EntityFrameworkCore/Providers/Sql/BaseSqlDialect.cs diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/BaseSqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/BaseSqlDialect.cs new file mode 100644 index 0000000..aefe79f --- /dev/null +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/BaseSqlDialect.cs @@ -0,0 +1,110 @@ +using System.Runtime.CompilerServices; +using Atomizer.EntityFrameworkCore.Entities; + +namespace Atomizer.EntityFrameworkCore.Providers.Sql; + +internal abstract class BaseSqlDialect : ISqlDialect +{ + // Job table and columns + protected readonly string _jTable; + protected readonly string _jId; + protected readonly string _jQueueKey; + protected readonly string _jPayloadType; + protected readonly string _jPayload; + protected readonly string _jScheduledAt; + protected readonly string _jVisibleAt; + protected readonly string _jStatus; + protected readonly string _jAttempts; + protected readonly string _jRetryIntervals; + protected readonly string _jCreatedAt; + protected readonly string _jUpdatedAt; + protected readonly string _jLeaseToken; + protected readonly string _jScheduleJobKey; + protected readonly string _jIdempotencyKey; + protected readonly string _jPartitionKey; + protected readonly string _jSequenceNumber; + + // Schedule table and columns + protected readonly string _sTable; + protected readonly string _sId; + protected readonly string _sJobKey; + protected readonly string _sQueueKey; + protected readonly string _sPayloadType; + protected readonly string _sPayload; + protected readonly string _sSchedule; + protected readonly string _sTimeZone; + protected readonly string _sMisfirePolicy; + protected readonly string _sMaxCatchUp; + protected readonly string _sEnabled; + protected readonly string _sRetryIntervals; + protected readonly string _sNextRunAt; + protected readonly string _sLastEnqueueAt; + protected readonly string _sCreatedAt; + protected readonly string _sUpdatedAt; + + protected readonly int _statusPending = (int)AtomizerEntityJobStatus.Pending; + protected readonly int _statusProcessing = (int)AtomizerEntityJobStatus.Processing; + + protected BaseSqlDialect(EntityMap jobs, EntityMap schedules) + { + var jc = jobs.Col; + _jTable = jobs.Table; + _jId = jc[nameof(AtomizerJobEntity.Id)]; + _jQueueKey = jc[nameof(AtomizerJobEntity.QueueKey)]; + _jPayloadType = jc[nameof(AtomizerJobEntity.PayloadType)]; + _jPayload = jc[nameof(AtomizerJobEntity.Payload)]; + _jScheduledAt = jc[nameof(AtomizerJobEntity.ScheduledAt)]; + _jVisibleAt = jc[nameof(AtomizerJobEntity.VisibleAt)]; + _jStatus = jc[nameof(AtomizerJobEntity.Status)]; + _jAttempts = jc[nameof(AtomizerJobEntity.Attempts)]; + _jRetryIntervals = jc[nameof(AtomizerJobEntity.RetryIntervals)]; + _jCreatedAt = jc[nameof(AtomizerJobEntity.CreatedAt)]; + _jUpdatedAt = jc[nameof(AtomizerJobEntity.UpdatedAt)]; + _jLeaseToken = jc[nameof(AtomizerJobEntity.LeaseToken)]; + _jScheduleJobKey = jc[nameof(AtomizerJobEntity.ScheduleJobKey)]; + _jIdempotencyKey = jc[nameof(AtomizerJobEntity.IdempotencyKey)]; + _jPartitionKey = jc[nameof(AtomizerJobEntity.PartitionKey)]; + _jSequenceNumber = jc[nameof(AtomizerJobEntity.SequenceNumber)]; + + var sc = schedules.Col; + _sTable = schedules.Table; + _sId = sc[nameof(AtomizerScheduleEntity.Id)]; + _sJobKey = sc[nameof(AtomizerScheduleEntity.JobKey)]; + _sQueueKey = sc[nameof(AtomizerScheduleEntity.QueueKey)]; + _sPayloadType = sc[nameof(AtomizerScheduleEntity.PayloadType)]; + _sPayload = sc[nameof(AtomizerScheduleEntity.Payload)]; + _sSchedule = sc[nameof(AtomizerScheduleEntity.Schedule)]; + _sTimeZone = sc[nameof(AtomizerScheduleEntity.TimeZone)]; + _sMisfirePolicy = sc[nameof(AtomizerScheduleEntity.MisfirePolicy)]; + _sMaxCatchUp = sc[nameof(AtomizerScheduleEntity.MaxCatchUp)]; + _sEnabled = sc[nameof(AtomizerScheduleEntity.Enabled)]; + _sRetryIntervals = sc[nameof(AtomizerScheduleEntity.RetryIntervals)]; + _sNextRunAt = sc[nameof(AtomizerScheduleEntity.NextRunAt)]; + _sLastEnqueueAt = sc[nameof(AtomizerScheduleEntity.LastEnqueueAt)]; + _sCreatedAt = sc[nameof(AtomizerScheduleEntity.CreatedAt)]; + _sUpdatedAt = sc[nameof(AtomizerScheduleEntity.UpdatedAt)]; + } + + protected static string SerializeIntervals(TimeSpan[] intervals) => + string.Join(";", Array.ConvertAll(intervals, ts => (long)ts.TotalMilliseconds)); + + public FormattableString ReleaseLeasedJobs(LeaseToken leaseToken, DateTimeOffset now) + { + var format = + $$""" + UPDATE {{_jTable}} + SET {{_jStatus}} = {{_statusPending}}, + {{_jLeaseToken}} = NULL, + {{_jVisibleAt}} = NULL, + {{_jUpdatedAt}} = {0} + WHERE {{_jLeaseToken}} = {1} + AND {{_jStatus}} = {{_statusProcessing}}; + """; + return FormattableStringFactory.Create(format, now, leaseToken.Token); + } + + public abstract FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int batchSize); + public abstract FormattableString InsertJobWithSequence(AtomizerJob job); + public abstract FormattableString GetDueSchedules(DateTimeOffset now); + public abstract FormattableString UpsertScheduleAsync(AtomizerSchedule schedule, DateTimeOffset now); +} diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs index b638884..9371a0e 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs @@ -3,85 +3,66 @@ namespace Atomizer.EntityFrameworkCore.Providers.Sql; -internal sealed class MySqlDialect : ISqlDialect +internal sealed class MySqlDialect(EntityMap jobs, EntityMap schedules) : BaseSqlDialect(jobs, schedules) { - private readonly EntityMap _jobs; - private readonly EntityMap _schedules; - - public MySqlDialect(EntityMap jobs, EntityMap schedules) + public override FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int batchSize) { - _jobs = jobs; - _schedules = schedules; - } - - public FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int batchSize) - { - var table = _jobs.Table; - var c = _jobs.Col; - var colStatus = c[nameof(AtomizerJobEntity.Status)]; - var colQueueKey = c[nameof(AtomizerJobEntity.QueueKey)]; - var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; - var colScheduledAt = c[nameof(AtomizerJobEntity.ScheduledAt)]; - var colId = c[nameof(AtomizerJobEntity.Id)]; - var colPartitionKey = c[nameof(AtomizerJobEntity.PartitionKey)]; - var colSequenceNumber = c[nameof(AtomizerJobEntity.SequenceNumber)]; - var colAttempts = c[nameof(AtomizerJobEntity.Attempts)]; - var statusPending = (int)AtomizerEntityJobStatus.Pending; - var statusProcessing = (int)AtomizerEntityJobStatus.Processing; var format = - $@"WITH blocked_partitions AS ( - SELECT DISTINCT {colPartitionKey} - FROM {table} - WHERE {colQueueKey} = {{0}} - AND {colPartitionKey} IS NOT NULL - AND ( - {colStatus} = {statusProcessing} - OR ({colStatus} = {statusPending} AND {colAttempts} > 0) - ) -), -partition_heads AS ( - SELECT j.{colPartitionKey}, MIN(j.{colSequenceNumber}) AS min_seq - FROM {table} AS j - LEFT JOIN blocked_partitions bp ON j.{colPartitionKey} = bp.{colPartitionKey} - WHERE j.{colQueueKey} = {{1}} - AND j.{colPartitionKey} IS NOT NULL - AND bp.{colPartitionKey} IS NULL - AND ( - (j.{colStatus} = {statusPending} - AND (j.{colVisibleAt} IS NULL OR j.{colVisibleAt} <= {{10}}) - AND j.{colScheduledAt} <= {{11}}) - OR (j.{colStatus} = {statusProcessing} AND j.{colVisibleAt} <= {{12}}) - ) - GROUP BY j.{colPartitionKey} -) -SELECT t.* -FROM {table} AS t -LEFT JOIN partition_heads ph - ON t.{colPartitionKey} = ph.{colPartitionKey} - AND t.{colSequenceNumber} = ph.min_seq -WHERE t.{colQueueKey} = {{2}} - AND ( - (t.{colPartitionKey} IS NULL - AND ( - (t.{colStatus} = {statusPending} - AND (t.{colVisibleAt} IS NULL OR t.{colVisibleAt} <= {{3}}) - AND t.{colScheduledAt} <= {{4}}) - OR (t.{colStatus} = {statusProcessing} AND t.{colVisibleAt} <= {{5}}) - ) - ) - OR - (t.{colPartitionKey} IS NOT NULL AND ph.min_seq IS NOT NULL - AND ( - (t.{colStatus} = {statusPending} - AND (t.{colVisibleAt} IS NULL OR t.{colVisibleAt} <= {{6}}) - AND t.{colScheduledAt} <= {{7}}) - OR (t.{colStatus} = {statusProcessing} AND t.{colVisibleAt} <= {{8}}) - ) - ) - ) -ORDER BY t.{colScheduledAt}, t.{colId} -LIMIT {{9}} -FOR UPDATE SKIP LOCKED;"; + $$""" + WITH blocked_partitions AS ( + SELECT DISTINCT {{_jPartitionKey}} + FROM {{_jTable}} + WHERE {{_jQueueKey}} = {0} + AND {{_jPartitionKey}} IS NOT NULL + AND ( + {{_jStatus}} = {{_statusProcessing}} + OR ({{_jStatus}} = {{_statusPending}} AND {{_jAttempts}} > 0) + ) + ), + partition_heads AS ( + SELECT j.{{_jPartitionKey}}, MIN(j.{{_jSequenceNumber}}) AS min_seq + FROM {{_jTable}} AS j + LEFT JOIN blocked_partitions bp ON j.{{_jPartitionKey}} = bp.{{_jPartitionKey}} + WHERE j.{{_jQueueKey}} = {1} + AND j.{{_jPartitionKey}} IS NOT NULL + AND bp.{{_jPartitionKey}} IS NULL + AND ( + (j.{{_jStatus}} = {{_statusPending}} + AND (j.{{_jVisibleAt}} IS NULL OR j.{{_jVisibleAt}} <= {10}) + AND j.{{_jScheduledAt}} <= {11}) + OR (j.{{_jStatus}} = {{_statusProcessing}} AND j.{{_jVisibleAt}} <= {12}) + ) + GROUP BY j.{{_jPartitionKey}} + ) + SELECT t.* + FROM {{_jTable}} AS t + LEFT JOIN partition_heads ph + ON t.{{_jPartitionKey}} = ph.{{_jPartitionKey}} + AND t.{{_jSequenceNumber}} = ph.min_seq + WHERE t.{{_jQueueKey}} = {2} + AND ( + (t.{{_jPartitionKey}} IS NULL + AND ( + (t.{{_jStatus}} = {{_statusPending}} + AND (t.{{_jVisibleAt}} IS NULL OR t.{{_jVisibleAt}} <= {3}) + AND t.{{_jScheduledAt}} <= {4}) + OR (t.{{_jStatus}} = {{_statusProcessing}} AND t.{{_jVisibleAt}} <= {5}) + ) + ) + OR + (t.{{_jPartitionKey}} IS NOT NULL AND ph.min_seq IS NOT NULL + AND ( + (t.{{_jStatus}} = {{_statusPending}} + AND (t.{{_jVisibleAt}} IS NULL OR t.{{_jVisibleAt}} <= {6}) + AND t.{{_jScheduledAt}} <= {7}) + OR (t.{{_jStatus}} = {{_statusProcessing}} AND t.{{_jVisibleAt}} <= {8}) + ) + ) + ) + ORDER BY t.{{_jScheduledAt}}, t.{{_jId}} + LIMIT {9} + FOR UPDATE SKIP LOCKED; + """; return FormattableStringFactory.Create( format, queueKey.Key, // {0} blocked_partitions queue filter @@ -100,45 +81,26 @@ LEFT JOIN partition_heads ph ); } - public FormattableString InsertJobWithSequence(AtomizerJob job) + public override FormattableString InsertJobWithSequence(AtomizerJob job) { var entity = job.ToEntity(); - var table = _jobs.Table; - var c = _jobs.Col; - var colId = c[nameof(AtomizerJobEntity.Id)]; - var colQueueKey = c[nameof(AtomizerJobEntity.QueueKey)]; - var colPayloadType = c[nameof(AtomizerJobEntity.PayloadType)]; - var colPayload = c[nameof(AtomizerJobEntity.Payload)]; - var colScheduledAt = c[nameof(AtomizerJobEntity.ScheduledAt)]; - var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; - var colStatus = c[nameof(AtomizerJobEntity.Status)]; - var colAttempts = c[nameof(AtomizerJobEntity.Attempts)]; - var colRetryIntervals = c[nameof(AtomizerJobEntity.RetryIntervals)]; - var colCreatedAt = c[nameof(AtomizerJobEntity.CreatedAt)]; - var colUpdatedAt = c[nameof(AtomizerJobEntity.UpdatedAt)]; - var colLeaseToken = c[nameof(AtomizerJobEntity.LeaseToken)]; - var colScheduleJobKey = c[nameof(AtomizerJobEntity.ScheduleJobKey)]; - var colIdempotencyKey = c[nameof(AtomizerJobEntity.IdempotencyKey)]; - var colPartitionKey = c[nameof(AtomizerJobEntity.PartitionKey)]; - var colSequenceNumber = c[nameof(AtomizerJobEntity.SequenceNumber)]; - var retryIntervals = string.Join( - ";", - Array.ConvertAll(entity.RetryIntervals, ts => (long)ts.TotalMilliseconds) - ); + var retryIntervals = SerializeIntervals(entity.RetryIntervals); var format = - $@"INSERT INTO {table} ( - {colId}, {colQueueKey}, {colPayloadType}, {colPayload}, - {colScheduledAt}, {colVisibleAt}, {colStatus}, {colAttempts}, - {colRetryIntervals}, {colCreatedAt}, {colUpdatedAt}, - {colLeaseToken}, {colScheduleJobKey}, {colIdempotencyKey}, - {colPartitionKey}, {colSequenceNumber} -) -SELECT {{0}}, {{1}}, {{2}}, {{3}}, - {{4}}, {{5}}, {{6}}, {{7}}, - {{8}}, {{9}}, {{10}}, - {{11}}, {{12}}, {{13}}, - {{14}}, - COALESCE((SELECT MAX(max_seq) FROM (SELECT MAX({colSequenceNumber}) AS max_seq FROM {table} WHERE {colQueueKey} = {{15}} AND {colPartitionKey} = {{16}}) AS sub), 0) + 1;"; + $$""" + INSERT INTO {{_jTable}} ( + {{_jId}}, {{_jQueueKey}}, {{_jPayloadType}}, {{_jPayload}}, + {{_jScheduledAt}}, {{_jVisibleAt}}, {{_jStatus}}, {{_jAttempts}}, + {{_jRetryIntervals}}, {{_jCreatedAt}}, {{_jUpdatedAt}}, + {{_jLeaseToken}}, {{_jScheduleJobKey}}, {{_jIdempotencyKey}}, + {{_jPartitionKey}}, {{_jSequenceNumber}} + ) + SELECT {0}, {1}, {2}, {3}, + {4}, {5}, {6}, {7}, + {8}, {9}, {10}, + {11}, {12}, {13}, + {14}, + COALESCE((SELECT MAX(max_seq) FROM (SELECT MAX({{_jSequenceNumber}}) AS max_seq FROM {{_jTable}} WHERE {{_jQueueKey}} = {15} AND {{_jPartitionKey}} = {16}) AS sub), 0) + 1; + """; return FormattableStringFactory.Create( format, entity.Id, @@ -161,114 +123,72 @@ public FormattableString InsertJobWithSequence(AtomizerJob job) ); } - public FormattableString ReleaseLeasedJobs(LeaseToken leaseToken, DateTimeOffset now) + public override FormattableString GetDueSchedules(DateTimeOffset now) { - var table = _jobs.Table; - var c = _jobs.Col; - var colStatus = c[nameof(AtomizerJobEntity.Status)]; - var colLeaseToken = c[nameof(AtomizerJobEntity.LeaseToken)]; - var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; - var colUpdatedAt = c[nameof(AtomizerJobEntity.UpdatedAt)]; - var statusPending = (int)AtomizerEntityJobStatus.Pending; - var statusProcessing = (int)AtomizerEntityJobStatus.Processing; var format = - $@"UPDATE {table} -SET {colStatus} = {statusPending}, - {colLeaseToken} = NULL, - {colVisibleAt} = NULL, - {colUpdatedAt} = {{0}} -WHERE {colLeaseToken} = {{1}} - AND {colStatus} = {statusProcessing};"; - return FormattableStringFactory.Create(format, now, leaseToken.Token); - } - - public FormattableString GetDueSchedules(DateTimeOffset now) - { - var table = _schedules.Table; - var c = _schedules.Col; - var colEnabled = c[nameof(AtomizerScheduleEntity.Enabled)]; - var colNextRunAt = c[nameof(AtomizerScheduleEntity.NextRunAt)]; - var colId = c[nameof(AtomizerScheduleEntity.Id)]; - var format = - $@"SELECT t.* -FROM {table} AS t -WHERE {colNextRunAt} <= {{0}} - AND {colEnabled} = TRUE -ORDER BY {colNextRunAt}, {colId} -FOR UPDATE SKIP LOCKED;"; + $$""" + SELECT t.* + FROM {{_sTable}} AS t + WHERE {{_sNextRunAt}} <= {0} + AND {{_sEnabled}} = TRUE + ORDER BY {{_sNextRunAt}}, {{_sId}} + FOR UPDATE SKIP LOCKED; + """; return FormattableStringFactory.Create(format, now); } - public FormattableString UpsertScheduleAsync(AtomizerSchedule schedule, DateTimeOffset now) + public override FormattableString UpsertScheduleAsync(AtomizerSchedule schedule, DateTimeOffset now) { var entity = schedule.ToEntity(); - var table = _schedules.Table; - var c = _schedules.Col; - var colId = c[nameof(AtomizerScheduleEntity.Id)]; - var colJobKey = c[nameof(AtomizerScheduleEntity.JobKey)]; - var colQueueKey = c[nameof(AtomizerScheduleEntity.QueueKey)]; - var colPayloadType = c[nameof(AtomizerScheduleEntity.PayloadType)]; - var colPayload = c[nameof(AtomizerScheduleEntity.Payload)]; - var colSchedule = c[nameof(AtomizerScheduleEntity.Schedule)]; - var colTimeZone = c[nameof(AtomizerScheduleEntity.TimeZone)]; - var colMisfirePolicy = c[nameof(AtomizerScheduleEntity.MisfirePolicy)]; - var colMaxCatchUp = c[nameof(AtomizerScheduleEntity.MaxCatchUp)]; - var colEnabled = c[nameof(AtomizerScheduleEntity.Enabled)]; - var colRetryIntervals = c[nameof(AtomizerScheduleEntity.RetryIntervals)]; - var colNextRunAt = c[nameof(AtomizerScheduleEntity.NextRunAt)]; - var colLastEnqueueAt = c[nameof(AtomizerScheduleEntity.LastEnqueueAt)]; - var colCreatedAt = c[nameof(AtomizerScheduleEntity.CreatedAt)]; - var colUpdatedAt = c[nameof(AtomizerScheduleEntity.UpdatedAt)]; - var retryIntervals = string.Join( - ";", - Array.ConvertAll(entity.RetryIntervals, ts => (long)ts.TotalMilliseconds) - ); + var retryIntervals = SerializeIntervals(entity.RetryIntervals); var format = - $@"INSERT INTO {table} ( - {colId}, - {colJobKey}, - {colQueueKey}, - {colPayloadType}, - {colPayload}, - {colSchedule}, - {colTimeZone}, - {colMisfirePolicy}, - {colMaxCatchUp}, - {colEnabled}, - {colRetryIntervals}, - {colNextRunAt}, - {colLastEnqueueAt}, - {colCreatedAt}, - {colUpdatedAt} -) VALUES ( - {{0}}, - {{1}}, - {{2}}, - {{3}}, - {{4}}, - {{5}}, - {{6}}, - {{7}}, - {{8}}, - {{9}}, - {{10}}, - {{11}}, - {{12}}, - {{13}}, - {{14}} -) -ON DUPLICATE KEY UPDATE - {colQueueKey} = VALUES({colQueueKey}), - {colPayloadType} = VALUES({colPayloadType}), - {colPayload} = VALUES({colPayload}), - {colSchedule} = VALUES({colSchedule}), - {colTimeZone} = VALUES({colTimeZone}), - {colMisfirePolicy} = VALUES({colMisfirePolicy}), - {colMaxCatchUp} = VALUES({colMaxCatchUp}), - {colEnabled} = VALUES({colEnabled}), - {colRetryIntervals} = VALUES({colRetryIntervals}), - {colNextRunAt} = VALUES({colNextRunAt}), - {colUpdatedAt} = {{14}};"; + $$""" + INSERT INTO {{_sTable}} ( + {{_sId}}, + {{_sJobKey}}, + {{_sQueueKey}}, + {{_sPayloadType}}, + {{_sPayload}}, + {{_sSchedule}}, + {{_sTimeZone}}, + {{_sMisfirePolicy}}, + {{_sMaxCatchUp}}, + {{_sEnabled}}, + {{_sRetryIntervals}}, + {{_sNextRunAt}}, + {{_sLastEnqueueAt}}, + {{_sCreatedAt}}, + {{_sUpdatedAt}} + ) VALUES ( + {0}, + {1}, + {2}, + {3}, + {4}, + {5}, + {6}, + {7}, + {8}, + {9}, + {10}, + {11}, + {12}, + {13}, + {14} + ) + ON DUPLICATE KEY UPDATE + {{_sQueueKey}} = VALUES({{_sQueueKey}}), + {{_sPayloadType}} = VALUES({{_sPayloadType}}), + {{_sPayload}} = VALUES({{_sPayload}}), + {{_sSchedule}} = VALUES({{_sSchedule}}), + {{_sTimeZone}} = VALUES({{_sTimeZone}}), + {{_sMisfirePolicy}} = VALUES({{_sMisfirePolicy}}), + {{_sMaxCatchUp}} = VALUES({{_sMaxCatchUp}}), + {{_sEnabled}} = VALUES({{_sEnabled}}), + {{_sRetryIntervals}} = VALUES({{_sRetryIntervals}}), + {{_sNextRunAt}} = VALUES({{_sNextRunAt}}), + {{_sUpdatedAt}} = {14}; + """; return FormattableStringFactory.Create( format, entity.Id, diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs index 3a588bb..470d449 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs @@ -3,84 +3,65 @@ namespace Atomizer.EntityFrameworkCore.Providers.Sql; -internal sealed class PostgreSqlDialect : ISqlDialect +internal sealed class PostgreSqlDialect(EntityMap jobs, EntityMap schedules) : BaseSqlDialect(jobs, schedules) { - private readonly EntityMap _jobs; - private readonly EntityMap _schedules; - - public PostgreSqlDialect(EntityMap jobs, EntityMap schedules) + public override FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int batchSize) { - _jobs = jobs; - _schedules = schedules; - } - - public FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int batchSize) - { - var table = _jobs.Table; - var c = _jobs.Col; - var colStatus = c[nameof(AtomizerJobEntity.Status)]; - var colQueueKey = c[nameof(AtomizerJobEntity.QueueKey)]; - var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; - var colScheduledAt = c[nameof(AtomizerJobEntity.ScheduledAt)]; - var colId = c[nameof(AtomizerJobEntity.Id)]; - var colPartitionKey = c[nameof(AtomizerJobEntity.PartitionKey)]; - var colSequenceNumber = c[nameof(AtomizerJobEntity.SequenceNumber)]; - var colAttempts = c[nameof(AtomizerJobEntity.Attempts)]; - var statusPending = (int)AtomizerEntityJobStatus.Pending; - var statusProcessing = (int)AtomizerEntityJobStatus.Processing; var format = - $@"WITH blocked_partitions AS ( - SELECT DISTINCT {colPartitionKey} - FROM {table} - WHERE {colQueueKey} = {{0}} - AND {colPartitionKey} IS NOT NULL - AND ( - {colStatus} = {statusProcessing} - OR ({colStatus} = {statusPending} AND {colAttempts} > 0) - ) -), -partition_heads AS ( - SELECT {colPartitionKey}, MIN({colSequenceNumber}) AS min_seq - FROM {table} - WHERE {colQueueKey} = {{1}} - AND {colPartitionKey} IS NOT NULL - AND {colPartitionKey} NOT IN (SELECT {colPartitionKey} FROM blocked_partitions) - AND ( - ({colStatus} = {statusPending} - AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{10}}) - AND {colScheduledAt} <= {{11}}) - OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{12}}) - ) - GROUP BY {colPartitionKey} -) -SELECT t.* -FROM {table} AS t -LEFT JOIN partition_heads ph - ON t.{colPartitionKey} = ph.{colPartitionKey} - AND t.{colSequenceNumber} = ph.min_seq -WHERE t.{colQueueKey} = {{2}} - AND ( - (t.{colPartitionKey} IS NULL - AND ( - (t.{colStatus} = {statusPending} - AND (t.{colVisibleAt} IS NULL OR t.{colVisibleAt} <= {{3}}) - AND t.{colScheduledAt} <= {{4}}) - OR (t.{colStatus} = {statusProcessing} AND t.{colVisibleAt} <= {{5}}) - ) - ) - OR - (t.{colPartitionKey} IS NOT NULL AND ph.min_seq IS NOT NULL - AND ( - (t.{colStatus} = {statusPending} - AND (t.{colVisibleAt} IS NULL OR t.{colVisibleAt} <= {{6}}) - AND t.{colScheduledAt} <= {{7}}) - OR (t.{colStatus} = {statusProcessing} AND t.{colVisibleAt} <= {{8}}) - ) - ) - ) -ORDER BY t.{colScheduledAt}, t.{colId} -LIMIT {{9}} -FOR NO KEY UPDATE SKIP LOCKED;"; + $$""" + WITH blocked_partitions AS ( + SELECT DISTINCT {{_jPartitionKey}} + FROM {{_jTable}} + WHERE {{_jQueueKey}} = {0} + AND {{_jPartitionKey}} IS NOT NULL + AND ( + {{_jStatus}} = {{_statusProcessing}} + OR ({{_jStatus}} = {{_statusPending}} AND {{_jAttempts}} > 0) + ) + ), + partition_heads AS ( + SELECT {{_jPartitionKey}}, MIN({{_jSequenceNumber}}) AS min_seq + FROM {{_jTable}} + WHERE {{_jQueueKey}} = {1} + AND {{_jPartitionKey}} IS NOT NULL + AND {{_jPartitionKey}} NOT IN (SELECT {{_jPartitionKey}} FROM blocked_partitions) + AND ( + ({{_jStatus}} = {{_statusPending}} + AND ({{_jVisibleAt}} IS NULL OR {{_jVisibleAt}} <= {10}) + AND {{_jScheduledAt}} <= {11}) + OR ({{_jStatus}} = {{_statusProcessing}} AND {{_jVisibleAt}} <= {12}) + ) + GROUP BY {{_jPartitionKey}} + ) + SELECT t.* + FROM {{_jTable}} AS t + LEFT JOIN partition_heads ph + ON t.{{_jPartitionKey}} = ph.{{_jPartitionKey}} + AND t.{{_jSequenceNumber}} = ph.min_seq + WHERE t.{{_jQueueKey}} = {2} + AND ( + (t.{{_jPartitionKey}} IS NULL + AND ( + (t.{{_jStatus}} = {{_statusPending}} + AND (t.{{_jVisibleAt}} IS NULL OR t.{{_jVisibleAt}} <= {3}) + AND t.{{_jScheduledAt}} <= {4}) + OR (t.{{_jStatus}} = {{_statusProcessing}} AND t.{{_jVisibleAt}} <= {5}) + ) + ) + OR + (t.{{_jPartitionKey}} IS NOT NULL AND ph.min_seq IS NOT NULL + AND ( + (t.{{_jStatus}} = {{_statusPending}} + AND (t.{{_jVisibleAt}} IS NULL OR t.{{_jVisibleAt}} <= {6}) + AND t.{{_jScheduledAt}} <= {7}) + OR (t.{{_jStatus}} = {{_statusProcessing}} AND t.{{_jVisibleAt}} <= {8}) + ) + ) + ) + ORDER BY t.{{_jScheduledAt}}, t.{{_jId}} + LIMIT {9} + FOR NO KEY UPDATE SKIP LOCKED; + """; return FormattableStringFactory.Create( format, queueKey.Key, // {0} blocked_partitions queue filter @@ -99,45 +80,26 @@ LEFT JOIN partition_heads ph ); } - public FormattableString InsertJobWithSequence(AtomizerJob job) + public override FormattableString InsertJobWithSequence(AtomizerJob job) { var entity = job.ToEntity(); - var table = _jobs.Table; - var c = _jobs.Col; - var colId = c[nameof(AtomizerJobEntity.Id)]; - var colQueueKey = c[nameof(AtomizerJobEntity.QueueKey)]; - var colPayloadType = c[nameof(AtomizerJobEntity.PayloadType)]; - var colPayload = c[nameof(AtomizerJobEntity.Payload)]; - var colScheduledAt = c[nameof(AtomizerJobEntity.ScheduledAt)]; - var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; - var colStatus = c[nameof(AtomizerJobEntity.Status)]; - var colAttempts = c[nameof(AtomizerJobEntity.Attempts)]; - var colRetryIntervals = c[nameof(AtomizerJobEntity.RetryIntervals)]; - var colCreatedAt = c[nameof(AtomizerJobEntity.CreatedAt)]; - var colUpdatedAt = c[nameof(AtomizerJobEntity.UpdatedAt)]; - var colLeaseToken = c[nameof(AtomizerJobEntity.LeaseToken)]; - var colScheduleJobKey = c[nameof(AtomizerJobEntity.ScheduleJobKey)]; - var colIdempotencyKey = c[nameof(AtomizerJobEntity.IdempotencyKey)]; - var colPartitionKey = c[nameof(AtomizerJobEntity.PartitionKey)]; - var colSequenceNumber = c[nameof(AtomizerJobEntity.SequenceNumber)]; - var retryIntervals = string.Join( - ";", - Array.ConvertAll(entity.RetryIntervals, ts => (long)ts.TotalMilliseconds) - ); + var retryIntervals = SerializeIntervals(entity.RetryIntervals); var format = - $@"INSERT INTO {table} ( - {colId}, {colQueueKey}, {colPayloadType}, {colPayload}, - {colScheduledAt}, {colVisibleAt}, {colStatus}, {colAttempts}, - {colRetryIntervals}, {colCreatedAt}, {colUpdatedAt}, - {colLeaseToken}, {colScheduleJobKey}, {colIdempotencyKey}, - {colPartitionKey}, {colSequenceNumber} -) -SELECT {{0}}, {{1}}, {{2}}, {{3}}, - {{4}}, {{5}}, {{6}}, {{7}}, - {{8}}, {{9}}, {{10}}, - {{11}}, {{12}}, {{13}}, - {{14}}, - COALESCE((SELECT MAX({colSequenceNumber}) FROM (SELECT {colSequenceNumber} FROM {table} WHERE {colQueueKey} = {{15}} AND {colPartitionKey} = {{16}}) AS sub), 0) + 1;"; + $$""" + INSERT INTO {{_jTable}} ( + {{_jId}}, {{_jQueueKey}}, {{_jPayloadType}}, {{_jPayload}}, + {{_jScheduledAt}}, {{_jVisibleAt}}, {{_jStatus}}, {{_jAttempts}}, + {{_jRetryIntervals}}, {{_jCreatedAt}}, {{_jUpdatedAt}}, + {{_jLeaseToken}}, {{_jScheduleJobKey}}, {{_jIdempotencyKey}}, + {{_jPartitionKey}}, {{_jSequenceNumber}} + ) + SELECT {0}, {1}, {2}, {3}, + {4}, {5}, {6}, {7}, + {8}, {9}, {10}, + {11}, {12}, {13}, + {14}, + COALESCE((SELECT MAX({{_jSequenceNumber}}) FROM (SELECT {{_jSequenceNumber}} FROM {{_jTable}} WHERE {{_jQueueKey}} = {15} AND {{_jPartitionKey}} = {16}) AS sub), 0) + 1; + """; return FormattableStringFactory.Create( format, entity.Id, @@ -160,114 +122,72 @@ public FormattableString InsertJobWithSequence(AtomizerJob job) ); } - public FormattableString ReleaseLeasedJobs(LeaseToken leaseToken, DateTimeOffset now) + public override FormattableString GetDueSchedules(DateTimeOffset now) { - var table = _jobs.Table; - var c = _jobs.Col; - var colStatus = c[nameof(AtomizerJobEntity.Status)]; - var colLeaseToken = c[nameof(AtomizerJobEntity.LeaseToken)]; - var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; - var colUpdatedAt = c[nameof(AtomizerJobEntity.UpdatedAt)]; - var statusPending = (int)AtomizerEntityJobStatus.Pending; - var statusProcessing = (int)AtomizerEntityJobStatus.Processing; var format = - $@"UPDATE {table} -SET {colStatus} = {statusPending}, - {colLeaseToken} = NULL, - {colVisibleAt} = NULL, - {colUpdatedAt} = {{0}} -WHERE {colLeaseToken} = {{1}} - AND {colStatus} = {statusProcessing};"; - return FormattableStringFactory.Create(format, now, leaseToken.Token); - } - - public FormattableString GetDueSchedules(DateTimeOffset now) - { - var table = _schedules.Table; - var c = _schedules.Col; - var colEnabled = c[nameof(AtomizerScheduleEntity.Enabled)]; - var colNextRunAt = c[nameof(AtomizerScheduleEntity.NextRunAt)]; - var colId = c[nameof(AtomizerScheduleEntity.Id)]; - var format = - $@"SELECT t.* -FROM {table} AS t -WHERE {colEnabled} = TRUE - AND {colNextRunAt} <= {{0}} -ORDER BY {colNextRunAt}, {colId} -FOR NO KEY UPDATE SKIP LOCKED;"; + $$""" + SELECT t.* + FROM {{_sTable}} AS t + WHERE {{_sEnabled}} = TRUE + AND {{_sNextRunAt}} <= {0} + ORDER BY {{_sNextRunAt}}, {{_sId}} + FOR NO KEY UPDATE SKIP LOCKED; + """; return FormattableStringFactory.Create(format, now); } - public FormattableString UpsertScheduleAsync(AtomizerSchedule schedule, DateTimeOffset now) + public override FormattableString UpsertScheduleAsync(AtomizerSchedule schedule, DateTimeOffset now) { var entity = schedule.ToEntity(); - var table = _schedules.Table; - var c = _schedules.Col; - var colId = c[nameof(AtomizerScheduleEntity.Id)]; - var colJobKey = c[nameof(AtomizerScheduleEntity.JobKey)]; - var colQueueKey = c[nameof(AtomizerScheduleEntity.QueueKey)]; - var colPayloadType = c[nameof(AtomizerScheduleEntity.PayloadType)]; - var colPayload = c[nameof(AtomizerScheduleEntity.Payload)]; - var colSchedule = c[nameof(AtomizerScheduleEntity.Schedule)]; - var colTimeZone = c[nameof(AtomizerScheduleEntity.TimeZone)]; - var colMisfirePolicy = c[nameof(AtomizerScheduleEntity.MisfirePolicy)]; - var colMaxCatchUp = c[nameof(AtomizerScheduleEntity.MaxCatchUp)]; - var colEnabled = c[nameof(AtomizerScheduleEntity.Enabled)]; - var colRetryIntervals = c[nameof(AtomizerScheduleEntity.RetryIntervals)]; - var colNextRunAt = c[nameof(AtomizerScheduleEntity.NextRunAt)]; - var colLastEnqueueAt = c[nameof(AtomizerScheduleEntity.LastEnqueueAt)]; - var colCreatedAt = c[nameof(AtomizerScheduleEntity.CreatedAt)]; - var colUpdatedAt = c[nameof(AtomizerScheduleEntity.UpdatedAt)]; - var retryIntervals = string.Join( - ";", - Array.ConvertAll(entity.RetryIntervals, ts => (long)ts.TotalMilliseconds) - ); + var retryIntervals = SerializeIntervals(entity.RetryIntervals); var format = - $@"INSERT INTO {table} ( - {colId}, - {colJobKey}, - {colQueueKey}, - {colPayloadType}, - {colPayload}, - {colSchedule}, - {colTimeZone}, - {colMisfirePolicy}, - {colMaxCatchUp}, - {colEnabled}, - {colRetryIntervals}, - {colNextRunAt}, - {colLastEnqueueAt}, - {colCreatedAt}, - {colUpdatedAt} -) VALUES ( - {{0}}, - {{1}}, - {{2}}, - {{3}}, - {{4}}, - {{5}}, - {{6}}, - {{7}}, - {{8}}, - {{9}}, - {{10}}, - {{11}}, - {{12}}, - {{13}}, - {{14}} -) -ON CONFLICT ({colJobKey}) DO UPDATE SET - {colQueueKey} = EXCLUDED.{colQueueKey}, - {colPayloadType} = EXCLUDED.{colPayloadType}, - {colPayload} = EXCLUDED.{colPayload}, - {colSchedule} = EXCLUDED.{colSchedule}, - {colTimeZone} = EXCLUDED.{colTimeZone}, - {colMisfirePolicy} = EXCLUDED.{colMisfirePolicy}, - {colMaxCatchUp} = EXCLUDED.{colMaxCatchUp}, - {colEnabled} = EXCLUDED.{colEnabled}, - {colRetryIntervals} = EXCLUDED.{colRetryIntervals}, - {colNextRunAt} = EXCLUDED.{colNextRunAt}, - {colUpdatedAt} = EXCLUDED.{colUpdatedAt};"; + $$""" + INSERT INTO {{_sTable}} ( + {{_sId}}, + {{_sJobKey}}, + {{_sQueueKey}}, + {{_sPayloadType}}, + {{_sPayload}}, + {{_sSchedule}}, + {{_sTimeZone}}, + {{_sMisfirePolicy}}, + {{_sMaxCatchUp}}, + {{_sEnabled}}, + {{_sRetryIntervals}}, + {{_sNextRunAt}}, + {{_sLastEnqueueAt}}, + {{_sCreatedAt}}, + {{_sUpdatedAt}} + ) VALUES ( + {0}, + {1}, + {2}, + {3}, + {4}, + {5}, + {6}, + {7}, + {8}, + {9}, + {10}, + {11}, + {12}, + {13}, + {14} + ) + ON CONFLICT ({{_sJobKey}}) DO UPDATE SET + {{_sQueueKey}} = EXCLUDED.{{_sQueueKey}}, + {{_sPayloadType}} = EXCLUDED.{{_sPayloadType}}, + {{_sPayload}} = EXCLUDED.{{_sPayload}}, + {{_sSchedule}} = EXCLUDED.{{_sSchedule}}, + {{_sTimeZone}} = EXCLUDED.{{_sTimeZone}}, + {{_sMisfirePolicy}} = EXCLUDED.{{_sMisfirePolicy}}, + {{_sMaxCatchUp}} = EXCLUDED.{{_sMaxCatchUp}}, + {{_sEnabled}} = EXCLUDED.{{_sEnabled}}, + {{_sRetryIntervals}} = EXCLUDED.{{_sRetryIntervals}}, + {{_sNextRunAt}} = EXCLUDED.{{_sNextRunAt}}, + {{_sUpdatedAt}} = EXCLUDED.{{_sUpdatedAt}}; + """; return FormattableStringFactory.Create( format, entity.Id, diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs index d941b67..465741c 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs @@ -3,82 +3,63 @@ namespace Atomizer.EntityFrameworkCore.Providers.Sql; -internal sealed class SqlServerDialect : ISqlDialect +internal sealed class SqlServerDialect(EntityMap jobs, EntityMap schedules) : BaseSqlDialect(jobs, schedules) { - private readonly EntityMap _jobs; - private readonly EntityMap _schedules; - - public SqlServerDialect(EntityMap jobs, EntityMap schedules) + public override FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int batchSize) { - _jobs = jobs; - _schedules = schedules; - } - - public FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int batchSize) - { - var table = _jobs.Table; - var c = _jobs.Col; - var colStatus = c[nameof(AtomizerJobEntity.Status)]; - var colQueueKey = c[nameof(AtomizerJobEntity.QueueKey)]; - var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; - var colScheduledAt = c[nameof(AtomizerJobEntity.ScheduledAt)]; - var colId = c[nameof(AtomizerJobEntity.Id)]; - var colPartitionKey = c[nameof(AtomizerJobEntity.PartitionKey)]; - var colSequenceNumber = c[nameof(AtomizerJobEntity.SequenceNumber)]; - var colAttempts = c[nameof(AtomizerJobEntity.Attempts)]; - var statusPending = (int)AtomizerEntityJobStatus.Pending; - var statusProcessing = (int)AtomizerEntityJobStatus.Processing; var format = - $@"WITH blocked_partitions AS ( - SELECT DISTINCT {colPartitionKey} - FROM {table} - WHERE {colQueueKey} = {{0}} - AND {colPartitionKey} IS NOT NULL - AND ( - {colStatus} = {statusProcessing} - OR ({colStatus} = {statusPending} AND {colAttempts} > 0) - ) -), -partition_heads AS ( - SELECT {colPartitionKey}, MIN({colSequenceNumber}) AS min_seq - FROM {table} - WHERE {colQueueKey} = {{1}} - AND {colPartitionKey} IS NOT NULL - AND {colPartitionKey} NOT IN (SELECT {colPartitionKey} FROM blocked_partitions) - AND ( - ({colStatus} = {statusPending} - AND ({colVisibleAt} IS NULL OR {colVisibleAt} <= {{9}}) - AND {colScheduledAt} <= {{10}}) - OR ({colStatus} = {statusProcessing} AND {colVisibleAt} <= {{11}}) - ) - GROUP BY {colPartitionKey} -) -SELECT TOP({batchSize}) t.* -FROM {table} AS t WITH (UPDLOCK, READPAST, ROWLOCK) -LEFT JOIN partition_heads ph - ON t.{colPartitionKey} = ph.{colPartitionKey} - AND t.{colSequenceNumber} = ph.min_seq -WHERE t.{colQueueKey} = {{2}} - AND ( - (t.{colPartitionKey} IS NULL - AND ( - (t.{colStatus} = {statusPending} - AND (t.{colVisibleAt} IS NULL OR t.{colVisibleAt} <= {{3}}) - AND t.{colScheduledAt} <= {{4}}) - OR (t.{colStatus} = {statusProcessing} AND t.{colVisibleAt} <= {{5}}) - ) - ) - OR - (t.{colPartitionKey} IS NOT NULL AND ph.min_seq IS NOT NULL - AND ( - (t.{colStatus} = {statusPending} - AND (t.{colVisibleAt} IS NULL OR t.{colVisibleAt} <= {{6}}) - AND t.{colScheduledAt} <= {{7}}) - OR (t.{colStatus} = {statusProcessing} AND t.{colVisibleAt} <= {{8}}) - ) - ) - ) -ORDER BY t.{colScheduledAt}, t.{colId};"; + $$""" + WITH blocked_partitions AS ( + SELECT DISTINCT {{_jPartitionKey}} + FROM {{_jTable}} + WHERE {{_jQueueKey}} = {0} + AND {{_jPartitionKey}} IS NOT NULL + AND ( + {{_jStatus}} = {{_statusProcessing}} + OR ({{_jStatus}} = {{_statusPending}} AND {{_jAttempts}} > 0) + ) + ), + partition_heads AS ( + SELECT {{_jPartitionKey}}, MIN({{_jSequenceNumber}}) AS min_seq + FROM {{_jTable}} + WHERE {{_jQueueKey}} = {1} + AND {{_jPartitionKey}} IS NOT NULL + AND {{_jPartitionKey}} NOT IN (SELECT {{_jPartitionKey}} FROM blocked_partitions) + AND ( + ({{_jStatus}} = {{_statusPending}} + AND ({{_jVisibleAt}} IS NULL OR {{_jVisibleAt}} <= {9}) + AND {{_jScheduledAt}} <= {10}) + OR ({{_jStatus}} = {{_statusProcessing}} AND {{_jVisibleAt}} <= {11}) + ) + GROUP BY {{_jPartitionKey}} + ) + SELECT TOP({{batchSize}}) t.* + FROM {{_jTable}} AS t WITH (UPDLOCK, READPAST, ROWLOCK) + LEFT JOIN partition_heads ph + ON t.{{_jPartitionKey}} = ph.{{_jPartitionKey}} + AND t.{{_jSequenceNumber}} = ph.min_seq + WHERE t.{{_jQueueKey}} = {2} + AND ( + (t.{{_jPartitionKey}} IS NULL + AND ( + (t.{{_jStatus}} = {{_statusPending}} + AND (t.{{_jVisibleAt}} IS NULL OR t.{{_jVisibleAt}} <= {3}) + AND t.{{_jScheduledAt}} <= {4}) + OR (t.{{_jStatus}} = {{_statusProcessing}} AND t.{{_jVisibleAt}} <= {5}) + ) + ) + OR + (t.{{_jPartitionKey}} IS NOT NULL AND ph.min_seq IS NOT NULL + AND ( + (t.{{_jStatus}} = {{_statusPending}} + AND (t.{{_jVisibleAt}} IS NULL OR t.{{_jVisibleAt}} <= {6}) + AND t.{{_jScheduledAt}} <= {7}) + OR (t.{{_jStatus}} = {{_statusProcessing}} AND t.{{_jVisibleAt}} <= {8}) + ) + ) + ) + ORDER BY t.{{_jScheduledAt}}, t.{{_jId}}; + """; return FormattableStringFactory.Create( format, queueKey.Key, // {0} blocked_partitions queue filter @@ -90,51 +71,32 @@ LEFT JOIN partition_heads ph now, // {6} partitioned VisibleAt now, // {7} partitioned ScheduledAt now, // {8} partitioned Processing VisibleAt - now, // {9} partition_heads VisibleAt (batchSize is TOP({batchSize}) inlined, not a placeholder) + now, // {9} partition_heads VisibleAt (batchSize is TOP(batchSize) inlined, not a placeholder) now, // {10} partition_heads ScheduledAt now // {11} partition_heads Processing VisibleAt ); } - public FormattableString InsertJobWithSequence(AtomizerJob job) + public override FormattableString InsertJobWithSequence(AtomizerJob job) { var entity = job.ToEntity(); - var table = _jobs.Table; - var c = _jobs.Col; - var colId = c[nameof(AtomizerJobEntity.Id)]; - var colQueueKey = c[nameof(AtomizerJobEntity.QueueKey)]; - var colPayloadType = c[nameof(AtomizerJobEntity.PayloadType)]; - var colPayload = c[nameof(AtomizerJobEntity.Payload)]; - var colScheduledAt = c[nameof(AtomizerJobEntity.ScheduledAt)]; - var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; - var colStatus = c[nameof(AtomizerJobEntity.Status)]; - var colAttempts = c[nameof(AtomizerJobEntity.Attempts)]; - var colRetryIntervals = c[nameof(AtomizerJobEntity.RetryIntervals)]; - var colCreatedAt = c[nameof(AtomizerJobEntity.CreatedAt)]; - var colUpdatedAt = c[nameof(AtomizerJobEntity.UpdatedAt)]; - var colLeaseToken = c[nameof(AtomizerJobEntity.LeaseToken)]; - var colScheduleJobKey = c[nameof(AtomizerJobEntity.ScheduleJobKey)]; - var colIdempotencyKey = c[nameof(AtomizerJobEntity.IdempotencyKey)]; - var colPartitionKey = c[nameof(AtomizerJobEntity.PartitionKey)]; - var colSequenceNumber = c[nameof(AtomizerJobEntity.SequenceNumber)]; - var retryIntervals = string.Join( - ";", - Array.ConvertAll(entity.RetryIntervals, ts => (long)ts.TotalMilliseconds) - ); + var retryIntervals = SerializeIntervals(entity.RetryIntervals); var format = - $@"INSERT INTO {table} ( - {colId}, {colQueueKey}, {colPayloadType}, {colPayload}, - {colScheduledAt}, {colVisibleAt}, {colStatus}, {colAttempts}, - {colRetryIntervals}, {colCreatedAt}, {colUpdatedAt}, - {colLeaseToken}, {colScheduleJobKey}, {colIdempotencyKey}, - {colPartitionKey}, {colSequenceNumber} -) -SELECT {{0}}, {{1}}, {{2}}, {{3}}, - {{4}}, {{5}}, {{6}}, {{7}}, - {{8}}, {{9}}, {{10}}, - {{11}}, {{12}}, {{13}}, - {{14}}, - COALESCE((SELECT MAX({colSequenceNumber}) FROM {table} WHERE {colQueueKey} = {{15}} AND {colPartitionKey} = {{16}}), 0) + 1;"; + $$""" + INSERT INTO {{_jTable}} ( + {{_jId}}, {{_jQueueKey}}, {{_jPayloadType}}, {{_jPayload}}, + {{_jScheduledAt}}, {{_jVisibleAt}}, {{_jStatus}}, {{_jAttempts}}, + {{_jRetryIntervals}}, {{_jCreatedAt}}, {{_jUpdatedAt}}, + {{_jLeaseToken}}, {{_jScheduleJobKey}}, {{_jIdempotencyKey}}, + {{_jPartitionKey}}, {{_jSequenceNumber}} + ) + SELECT {0}, {1}, {2}, {3}, + {4}, {5}, {6}, {7}, + {8}, {9}, {10}, + {11}, {12}, {13}, + {14}, + COALESCE((SELECT MAX({{_jSequenceNumber}}) FROM {{_jTable}} WHERE {{_jQueueKey}} = {15} AND {{_jPartitionKey}} = {16}), 0) + 1; + """; return FormattableStringFactory.Create( format, entity.Id, @@ -157,133 +119,91 @@ public FormattableString InsertJobWithSequence(AtomizerJob job) ); } - public FormattableString ReleaseLeasedJobs(LeaseToken leaseToken, DateTimeOffset now) + public override FormattableString GetDueSchedules(DateTimeOffset now) { - var table = _jobs.Table; - var c = _jobs.Col; - var colStatus = c[nameof(AtomizerJobEntity.Status)]; - var colLeaseToken = c[nameof(AtomizerJobEntity.LeaseToken)]; - var colVisibleAt = c[nameof(AtomizerJobEntity.VisibleAt)]; - var colUpdatedAt = c[nameof(AtomizerJobEntity.UpdatedAt)]; - var statusPending = (int)AtomizerEntityJobStatus.Pending; - var statusProcessing = (int)AtomizerEntityJobStatus.Processing; var format = - $@"UPDATE {table} -SET {colStatus} = {statusPending}, - {colLeaseToken} = NULL, - {colVisibleAt} = NULL, - {colUpdatedAt} = {{0}} -WHERE {colLeaseToken} = {{1}} - AND {colStatus} = {statusProcessing};"; - return FormattableStringFactory.Create(format, now, leaseToken.Token); - } - - public FormattableString GetDueSchedules(DateTimeOffset now) - { - var table = _schedules.Table; - var c = _schedules.Col; - var colEnabled = c[nameof(AtomizerScheduleEntity.Enabled)]; - var colNextRunAt = c[nameof(AtomizerScheduleEntity.NextRunAt)]; - var colId = c[nameof(AtomizerScheduleEntity.Id)]; - var format = - $@"SELECT t.* -FROM {table} AS t WITH (UPDLOCK, READPAST, ROWLOCK) -WHERE {colNextRunAt} <= {{0}} - AND {colEnabled} = 1 -ORDER BY {colNextRunAt}, {colId};"; + $$""" + SELECT t.* + FROM {{_sTable}} AS t WITH (UPDLOCK, READPAST, ROWLOCK) + WHERE {{_sNextRunAt}} <= {0} + AND {{_sEnabled}} = 1 + ORDER BY {{_sNextRunAt}}, {{_sId}}; + """; return FormattableStringFactory.Create(format, now); } - public FormattableString UpsertScheduleAsync(AtomizerSchedule schedule, DateTimeOffset now) + public override FormattableString UpsertScheduleAsync(AtomizerSchedule schedule, DateTimeOffset now) { var entity = schedule.ToEntity(); - var table = _schedules.Table; - var c = _schedules.Col; - var colId = c[nameof(AtomizerScheduleEntity.Id)]; - var colJobKey = c[nameof(AtomizerScheduleEntity.JobKey)]; - var colQueueKey = c[nameof(AtomizerScheduleEntity.QueueKey)]; - var colPayloadType = c[nameof(AtomizerScheduleEntity.PayloadType)]; - var colPayload = c[nameof(AtomizerScheduleEntity.Payload)]; - var colSchedule = c[nameof(AtomizerScheduleEntity.Schedule)]; - var colTimeZone = c[nameof(AtomizerScheduleEntity.TimeZone)]; - var colMisfirePolicy = c[nameof(AtomizerScheduleEntity.MisfirePolicy)]; - var colMaxCatchUp = c[nameof(AtomizerScheduleEntity.MaxCatchUp)]; - var colEnabled = c[nameof(AtomizerScheduleEntity.Enabled)]; - var colRetryIntervals = c[nameof(AtomizerScheduleEntity.RetryIntervals)]; - var colNextRunAt = c[nameof(AtomizerScheduleEntity.NextRunAt)]; - var colLastEnqueueAt = c[nameof(AtomizerScheduleEntity.LastEnqueueAt)]; - var colCreatedAt = c[nameof(AtomizerScheduleEntity.CreatedAt)]; - var colUpdatedAt = c[nameof(AtomizerScheduleEntity.UpdatedAt)]; - var retryIntervals = string.Join( - ";", - Array.ConvertAll(entity.RetryIntervals, ts => (long)ts.TotalMilliseconds) - ); + var retryIntervals = SerializeIntervals(entity.RetryIntervals); var format = - $@"MERGE {table} WITH (HOLDLOCK) AS target -USING (SELECT {{0}}) AS src ({colJobKey}) -ON target.{colJobKey} = src.{colJobKey} -WHEN MATCHED THEN UPDATE SET - {colQueueKey} = {{1}}, - {colPayloadType} = {{2}}, - {colPayload} = {{3}}, - {colSchedule} = {{4}}, - {colTimeZone} = {{5}}, - {colMisfirePolicy} = {{6}}, - {colMaxCatchUp} = {{7}}, - {colEnabled} = {{8}}, - {colRetryIntervals} = {{9}}, - {colNextRunAt} = {{10}}, - {colUpdatedAt} = {{11}} -WHEN NOT MATCHED THEN INSERT ( - {colId}, - {colJobKey}, - {colQueueKey}, - {colPayloadType}, - {colPayload}, - {colSchedule}, - {colTimeZone}, - {colMisfirePolicy}, - {colMaxCatchUp}, - {colEnabled}, - {colRetryIntervals}, - {colNextRunAt}, - {colLastEnqueueAt}, - {colCreatedAt}, - {colUpdatedAt} -) VALUES ( - {{12}}, - {{0}}, - {{1}}, - {{2}}, - {{3}}, - {{4}}, - {{5}}, - {{6}}, - {{7}}, - {{8}}, - {{9}}, - {{10}}, - {{13}}, - {{14}}, - {{11}} -);"; + $$""" + MERGE {{_sTable}} WITH (HOLDLOCK) AS target + USING (SELECT {0}) AS src ({{_sJobKey}}) + ON target.{{_sJobKey}} = src.{{_sJobKey}} + WHEN MATCHED THEN UPDATE SET + {{_sQueueKey}} = {1}, + {{_sPayloadType}} = {2}, + {{_sPayload}} = {3}, + {{_sSchedule}} = {4}, + {{_sTimeZone}} = {5}, + {{_sMisfirePolicy}} = {6}, + {{_sMaxCatchUp}} = {7}, + {{_sEnabled}} = {8}, + {{_sRetryIntervals}} = {9}, + {{_sNextRunAt}} = {10}, + {{_sUpdatedAt}} = {11} + WHEN NOT MATCHED THEN INSERT ( + {{_sId}}, + {{_sJobKey}}, + {{_sQueueKey}}, + {{_sPayloadType}}, + {{_sPayload}}, + {{_sSchedule}}, + {{_sTimeZone}}, + {{_sMisfirePolicy}}, + {{_sMaxCatchUp}}, + {{_sEnabled}}, + {{_sRetryIntervals}}, + {{_sNextRunAt}}, + {{_sLastEnqueueAt}}, + {{_sCreatedAt}}, + {{_sUpdatedAt}} + ) VALUES ( + {12}, + {0}, + {1}, + {2}, + {3}, + {4}, + {5}, + {6}, + {7}, + {8}, + {9}, + {10}, + {13}, + {14}, + {11} + ); + """; return FormattableStringFactory.Create( format, - entity.JobKey, // {0} - entity.QueueKey, // {1} - entity.PayloadType, // {2} - entity.Payload, // {3} - entity.Schedule, // {4} - entity.TimeZone, // {5} + entity.JobKey, // {0} + entity.QueueKey, // {1} + entity.PayloadType, // {2} + entity.Payload, // {3} + entity.Schedule, // {4} + entity.TimeZone, // {5} (int)entity.MisfirePolicy, // {6} - entity.MaxCatchUp, // {7} + entity.MaxCatchUp, // {7} entity.Enabled ? 1 : 0, // {8} - retryIntervals, // {9} - entity.NextRunAt, // {10} - now, // {11} - entity.Id, // {12} + retryIntervals, // {9} + entity.NextRunAt, // {10} + now, // {11} + entity.Id, // {12} entity.LastEnqueueAt, // {13} - entity.CreatedAt // {14} + entity.CreatedAt // {14} ); } } From 962f993f56ab4d17ecc3ef69aa120b557f3cbaf2 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 20:24:33 +0200 Subject: [PATCH 49/53] chore: remove tracked .planning files from git history MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These files should never have been committed — .planning/ is already in .gitignore. Untracking all four summary files that slipped through. Co-Authored-By: Claude Sonnet 4.6 --- .../08-01-SUMMARY.md | 125 --------------- .../08-03-SUMMARY.md | 81 ---------- .../09-01-SUMMARY.md | 95 ------------ .../09-02-SUMMARY.md | 142 ------------------ 4 files changed, 443 deletions(-) delete mode 100644 .planning/phases/08-inmemory-backend-unit-tests/08-01-SUMMARY.md delete mode 100644 .planning/phases/08-inmemory-backend-unit-tests/08-03-SUMMARY.md delete mode 100644 .planning/phases/09-ef-core-backend-integration-tests/09-01-SUMMARY.md delete mode 100644 .planning/phases/09-ef-core-backend-integration-tests/09-02-SUMMARY.md diff --git a/.planning/phases/08-inmemory-backend-unit-tests/08-01-SUMMARY.md b/.planning/phases/08-inmemory-backend-unit-tests/08-01-SUMMARY.md deleted file mode 100644 index 5aa4a2a..0000000 --- a/.planning/phases/08-inmemory-backend-unit-tests/08-01-SUMMARY.md +++ /dev/null @@ -1,125 +0,0 @@ ---- -phase: 08-inmemory-backend-unit-tests -plan: "01" -subsystem: storage -tags: [fifo, inmemory, idempotency, partition-blocking, tdd] -dependency_graph: - requires: [] - provides: [InMemoryStorage-FIFO, ScheduleProcessor-PartitionKey] - affects: [08-02-PLAN.md] -tech_stack: - added: [] - patterns: - - ConcurrentDictionary nested per-queue partition sequence counter - - Three-pass FIFO filter in GetDueJobsAsync (blockedPartitions + eligible + partitionHeads) - - netstandard2.0-safe OrderBy().First() instead of MinBy() -key_files: - created: [] - modified: - - src/Atomizer/Storage/InMemoryStorage.cs - - src/Atomizer/Scheduling/ScheduleProcessor.cs - - tests/Atomizer.Tests/Storage/InMemoryStorageTests.cs -decisions: - - CR-01 idempotency check placed before sequence assignment so no sequence number is consumed for a duplicate - - Used ConcurrentDictionary> for per-(queue, partitionKey) sequences - - Used job.PartitionKey.Key (string) as inner dict key to avoid value-object boxing - - OrderBy(SequenceNumber).First() used instead of MinBy() for netstandard2.0 compatibility -metrics: - duration: "4m 21s" - completed: "2026-05-04" - tasks_completed: 3 - files_modified: 3 ---- - -# Phase 8 Plan 1: FIFO InMemoryStorage Implementation Summary - -**One-liner:** FIFO sequence assignment and partition blocking in InMemoryStorage with full CR-01 idempotency fix, plus ScheduleProcessor partitionKey forwarding. - -## What Was Built - -Implemented FIFO ordering and partition blocking in `InMemoryStorage` (satisfying FIFO-10), and wired `ScheduleProcessor` to pass `schedule.PartitionKey` into `AtomizerJob.Create()` (D-07). - -### Task 1: InsertAsync FIFO with idempotency fix (TDD) - -Added `_partitionSequences` field (`ConcurrentDictionary>`) alongside `_queues`. Replaced the flat `InsertAsync` body with a three-step implementation: - -1. **CR-01 idempotency check** — linear scan of `_jobs.Values` for a matching `IdempotencyKey`. On collision: assigns `existing.SequenceNumber` to the passed-in job and returns `existing.Id` without touching `_queues`, `_leasesByToken`, or `EvictCompletedAndFailed`. -2. **FIFO-09 sequence assignment** — `_partitionSequences.GetOrAdd` per queue, then `AddOrUpdate` for atomic increment per partition key. Sequence starts at 1. Unpartitioned jobs (`PartitionKey == null`) leave `SequenceNumber` null. -3. **Store + index** — unchanged from original. - -### Task 2: GetDueJobsAsync three-pass FIFO filter (TDD) - -Replaced the single-pass LINQ chain with three passes: - -- **Pass 1:** Build `HashSet blockedPartitions` by scanning all `ids.Keys` for jobs where `IsPartitionBlocked == true`. Uses the domain property rather than re-implementing the condition. -- **Pass 2:** Filter eligible candidates using the existing status/time logic, plus exclude jobs whose `PartitionKey.Key` is in `blockedPartitions`. -- **Pass 3:** Split eligible into `unpartitioned` and `partitionHeads`. Partition heads selected via `GroupBy(PartitionKey.Key).Select(g => g.OrderBy(SequenceNumber).First())` — netstandard2.0-safe alternative to `MinBy()`. Concat both, order by `ScheduledAt`/`CreatedAt`, take `batchSize`. - -### Task 3: ScheduleProcessor PartitionKey forwarding - -Added `partitionKey: schedule.PartitionKey` as the final named argument to `AtomizerJob.Create()` in `ScheduleProcessor.ProcessAsync`. One-line change. - -## Deviations from Plan - -### Auto-fixed Issues - -**1. [Rule 1 - Bug] CSharpier formatting violation in InMemoryStorage.cs** -- **Found during:** Task 3 verification (CSharpier check) -- **Issue:** Two long lines in `InsertAsync` (GetOrAdd and AddOrUpdate calls) exceeded printWidth: 120 -- **Fix:** Ran `csharpier format` on the file; wrapped GetOrAdd call across 3 lines -- **Files modified:** `src/Atomizer/Storage/InMemoryStorage.cs` -- **Commit:** 691d71a - -## TDD Gate Compliance - -| Task | RED commit | GREEN commit | -|------|-----------|-------------| -| Task 1 (InsertAsync) | 0846a12 — 5 failing tests | 1d7588a — all pass | -| Task 2 (GetDueJobsAsync) | 2abf01d — 2 failing tests | aa161ea — all pass | - -Both RED and GREEN gates satisfied. RED commits preceded GREEN commits. - -## Test Coverage - -11 new unit tests added to `InMemoryStorageTests.cs`: - -**InsertAsync FIFO tests (Task 1):** -- `InsertAsync_WhenPartitionedJob_ShouldAssignSequenceNumberStartingAtOne` -- `InsertAsync_WhenPartitionedJobsInDifferentQueues_ShouldAssignIndependentSequences` -- `InsertAsync_WhenUnpartitionedJob_ShouldLeaveSequenceNumberNull` -- `InsertAsync_WhenIdempotencyKeyCollision_ShouldReturnExistingIdAndAssignExistingSequenceNumber` -- `InsertAsync_WhenIdempotencyKeyCollision_ShouldNotIncreaseJobCount` - -**GetDueJobsAsync FIFO tests (Task 2):** -- `GetDueJobsAsync_WhenTwoJobsSharePartition_ShouldReturnOnlyLowestSequenceNumber` -- `GetDueJobsAsync_WhenPartitionHeadAndUnpartitionedJobExist_ShouldReturnBoth` -- `GetDueJobsAsync_WhenPartitionJobIsProcessing_ShouldReturnEmpty` -- `GetDueJobsAsync_WhenPartitionJobIsPendingWithAttempts_ShouldReturnEmpty` -- `GetDueJobsAsync_WhenQueueABlockedPartitionSameKeyAsQueueB_ShouldReturnQueueBJobUnaffected` -- `GetDueJobsAsync_WhenProcessingJobHasExpiredVisibleAt_ShouldReturnIt` - -## Verification Results - -``` -dotnet build — 0 errors (pre-existing NU1903 warnings only) -dotnet test tests/Atomizer.Tests — 104/104 passed (net8.0 + net10.0) -csharpier check — all files formatted -``` - -## Known Stubs - -None — all data paths are fully wired. - -## Threat Flags - -None — all changes are internal in-process storage; no new network endpoints, public API surface, or trust boundary crossings introduced. - -## Self-Check: PASSED - -- `src/Atomizer/Storage/InMemoryStorage.cs` contains `_partitionSequences` field: confirmed -- `InsertAsync` contains `_jobs.Values.FirstOrDefault(j => j.IdempotencyKey == job.IdempotencyKey)`: confirmed -- `InsertAsync` contains `_partitionSequences.GetOrAdd` and `partitionSequences.AddOrUpdate`: confirmed -- `GetDueJobsAsync` contains `blockedPartitions` HashSet and `IsPartitionBlocked` usage: confirmed -- `GetDueJobsAsync` contains `partitionHeads` and `unpartitioned` variables: confirmed -- `ScheduleProcessor.cs` contains `partitionKey: schedule.PartitionKey`: confirmed -- Commits exist: 0846a12, 1d7588a, 2abf01d, aa161ea, 691d71a diff --git a/.planning/phases/08-inmemory-backend-unit-tests/08-03-SUMMARY.md b/.planning/phases/08-inmemory-backend-unit-tests/08-03-SUMMARY.md deleted file mode 100644 index a7ac7f7..0000000 --- a/.planning/phases/08-inmemory-backend-unit-tests/08-03-SUMMARY.md +++ /dev/null @@ -1,81 +0,0 @@ ---- -phase: 08-inmemory-backend-unit-tests -plan: "03" -subsystem: tests -tags: [fifo, contract-tests, terminal-state, unblocking, fifo-13] -dependency_graph: - requires: [08-02-PLAN.md] - provides: [FIFO-13 terminal-state unblocking coverage] - affects: [AtomizerStorageContractTests, InMemoryStorageContractTests] -tech_stack: - added: [] - patterns: [xUnit Fact, domain-method-chain (Lease→Attempt→MarkAsCompleted/MarkAsFailed)] -key_files: - created: [] - modified: - - tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs -decisions: - - "Used domain method chain (Lease→Attempt→MarkAsCompleted/MarkAsFailed) in test body to transition job1 to terminal state, consistent with existing test patterns" -metrics: - duration: "~4 minutes" - completed: "2026-05-04" - tasks_completed: 1 - tasks_total: 1 - files_changed: 1 ---- - -# Phase 08 Plan 03: Terminal-State Unblocking Tests Summary - -Two contract tests added to AtomizerStorageContractTests covering FIFO-13 terminal-state unblocking: Completed and Failed head jobs release their partition for the next job. - -## Tasks Completed - -| Task | Name | Commit | Files | -|------|------|--------|-------| -| 1 | Add terminal-unblocking tests to AtomizerStorageContractTests | ba8b01c | tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs | - -## What Was Built - -Added two `[Fact]` methods to `AtomizerStorageContractTests` under a `// FIFO-13: terminal-state unblocking` comment block, inserted immediately before the `// Helper` region: - -1. **`GetDueJobsAsync_WhenPartitionHeadCompleted_ShouldUnblockNextJob`** — inserts two partitioned jobs, transitions job1 to `Completed` via `Lease → Attempt → MarkAsCompleted`, calls `UpdateJobsAsync`, then asserts `GetDueJobsAsync` returns job2. - -2. **`GetDueJobsAsync_WhenPartitionHeadFailed_ShouldUnblockNextJob`** — same structure but transitions job1 to `Failed` via `Lease → Attempt → MarkAsFailed`. - -Both tests are inherited by `InMemoryStorageContractTests` (and all future backend contract subclasses) without any additional code changes. - -## Verification Results - -- `InMemoryStorageContractTests` now runs **11 tests** (was 9 before this plan). -- **net8.0**: 11/11 passed. -- **net10.0**: 11/11 passed. -- net6.0 test host failed to launch (pre-existing environment issue — .NET 6 runtime not installed on this machine; 0 build errors). -- Both new tests confirmed passing in both supported runtimes. - -Acceptance criteria: - -``` -grep -c "GetDueJobsAsync_WhenPartitionHeadCompleted_ShouldUnblockNextJob\|GetDueJobsAsync_WhenPartitionHeadFailed_ShouldUnblockNextJob" → 2 -grep -c "MarkAsCompleted\|MarkAsFailed" → 4 -``` - -Both output as expected. - -## Deviations from Plan - -None — plan executed exactly as written. - -## Known Stubs - -None. - -## Threat Flags - -None — test utilities file only; no new production code, network endpoints, auth paths, or schema changes introduced. - -## Self-Check: PASSED - -- [x] `tests/Atomizer.Tests.Utilities/StorageContract/AtomizerStorageContractTests.cs` modified — verified -- [x] Commit `ba8b01c` exists — verified -- [x] Both new test methods present (grep count = 2) — verified -- [x] All 11 InMemoryStorageContractTests pass on net8.0 and net10.0 — verified diff --git a/.planning/phases/09-ef-core-backend-integration-tests/09-01-SUMMARY.md b/.planning/phases/09-ef-core-backend-integration-tests/09-01-SUMMARY.md deleted file mode 100644 index e5072b8..0000000 --- a/.planning/phases/09-ef-core-backend-integration-tests/09-01-SUMMARY.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -phase: 09-ef-core-backend-integration-tests -plan: "01" -subsystem: ef-core-entity-layer -tags: [ef-core, entity, fifo, partition-key, sequence-number, sql-dialect] -dependency_graph: - requires: [] - provides: - - AtomizerJobEntity.PartitionKey (string?) - - AtomizerJobEntity.SequenceNumber (long?) - - AtomizerJobEntityMapper round-trip for PartitionKey and SequenceNumber - - AtomizerJobEntityConfiguration column config for both new columns - - ISqlDialect.InsertJobWithSequence method signature - affects: - - src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs - - src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs - - src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs -tech_stack: - added: [] - patterns: - - EF Core nullable column configuration via HasMaxLength(255).IsRequired(false) - - ISqlDialect strategy interface extension for provider-specific SQL -key_files: - created: [] - modified: - - src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs - - src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs - - src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs -decisions: - - ISqlDialect.InsertJobWithSequence added as interface-only contract; dialect implementations deferred to Plan 02 (expected CS0535 build failures are intentional) - - No explicit HasColumnType on SequenceNumber — EF Core auto-maps long? to bigint on all three supported providers -metrics: - duration: "2m" - completed: "2026-05-04" - tasks_completed: 2 - tasks_total: 2 - files_changed: 3 ---- - -# Phase 09 Plan 01: EF Core Entity Layer FIFO Foundation Summary - -EF Core entity, mapper, configuration, and ISqlDialect interface extended with PartitionKey (string?) and SequenceNumber (long?) as the foundational contracts for FIFO processing. - -## Tasks Completed - -| Task | Name | Commit | Files | -|------|------|--------|-------| -| 1 | Add PartitionKey and SequenceNumber to AtomizerJobEntity and both mapper directions | 4e792da | src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs | -| 2 | Configure new columns in AtomizerJobEntityConfiguration and add InsertJobWithSequence to ISqlDialect | 5004096 | src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs, src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs | - -## What Was Built - -### AtomizerJobEntity (Task 1) -- Added `public string? PartitionKey { get; set; }` with XML doc after `IdempotencyKey` -- Added `public long? SequenceNumber { get; set; }` with XML doc before `Errors` navigation property -- `ToEntity` mapper: `PartitionKey = job.PartitionKey?.ToString()` and `SequenceNumber = job.SequenceNumber` -- `ToAtomizerJob` mapper: `PartitionKey = entity.PartitionKey != null ? new PartitionKey(entity.PartitionKey) : null` and `SequenceNumber = entity.SequenceNumber` - -### AtomizerJobEntityConfiguration (Task 2) -- `builder.Property(job => job.PartitionKey).HasMaxLength(255).IsRequired(false)` — enforces same 255-char cap as the PartitionKey value object constructor (T-09-01 mitigation) -- `builder.Property(job => job.SequenceNumber).IsRequired(false)` — EF Core auto-maps long? to bigint - -### ISqlDialect (Task 2) -- Added `FormattableString InsertJobWithSequence(AtomizerJob job)` with XML doc -- Dialect implementations (PostgreSqlDialect, SqlServerDialect, MySqlDialect) will implement this in Plan 02 - -## Deviations from Plan - -None — plan executed exactly as written. The expected CS0535 build failures on all three dialect classes are confirmed and correct per the plan's verification section. - -## Verification Results - -All grep checks pass: -- `grep -c "public string? PartitionKey" AtomizerJobEntity.cs` → 1 -- `grep -c "public long? SequenceNumber" AtomizerJobEntity.cs` → 1 -- `grep -c "InsertJobWithSequence" ISqlDialect.cs` → 1 -- `grep -c "HasMaxLength(255)" AtomizerJobEntityConfiguration.cs` → 1 - -Build status: Expected CS0535 failures on MySqlDialect, PostgreSqlDialect, SqlServerDialect (dialect implementations deferred to Plan 02). - -## Known Stubs - -None — this plan establishes contracts only; no stub data flows to UI rendering. - -## Threat Surface Scan - -No new network endpoints, auth paths, file access patterns, or schema changes at trust boundaries beyond what the plan's threat model covers. - -## Self-Check: PASSED - -- 4e792da: feat(09-01): add PartitionKey and SequenceNumber to AtomizerJobEntity — FOUND -- 5004096: feat(09-01): configure PartitionKey/SequenceNumber columns and add InsertJobWithSequence to ISqlDialect — FOUND -- src/Atomizer.EntityFrameworkCore/Entities/AtomizerJobEntity.cs — exists, contains `public string? PartitionKey` -- src/Atomizer.EntityFrameworkCore/Configurations/AtomizerJobEntityConfiguration.cs — exists, contains `HasMaxLength(255)` -- src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs — exists, contains `InsertJobWithSequence` diff --git a/.planning/phases/09-ef-core-backend-integration-tests/09-02-SUMMARY.md b/.planning/phases/09-ef-core-backend-integration-tests/09-02-SUMMARY.md deleted file mode 100644 index 91a4897..0000000 --- a/.planning/phases/09-ef-core-backend-integration-tests/09-02-SUMMARY.md +++ /dev/null @@ -1,142 +0,0 @@ ---- -phase: 09-ef-core-backend-integration-tests -plan: "02" -subsystem: ef-core-storage -tags: - - fifo - - sql-dialects - - partitioned-insert - - cte - - sequence-number -dependency_graph: - requires: - - "09-01" - provides: - - "FIFO-aware GetDueJobs SQL (all three providers)" - - "InsertJobWithSequence SQL (all three providers)" - - "Partitioned insert branch in EntityFrameworkCoreStorage" - - "CR-01 idempotency SequenceNumber fix" - affects: - - "09-03" -tech_stack: - added: [] - patterns: - - "CTE blocked_partitions + partition_heads for FIFO-aware job acquisition" - - "COALESCE(MAX(SequenceNumber), 0) + 1 atomic sequence assignment per (queue, partition_key)" - - "Derived-table subquery form for MySQL and PostgreSQL INSERT...SELECT on same table" - - "Provider-specific locking: FOR NO KEY UPDATE SKIP LOCKED (PG), WITH (UPDLOCK,READPAST,ROWLOCK) on outer FROM (MSSQL), FOR UPDATE SKIP LOCKED (MySQL)" -key_files: - created: [] - modified: - - src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs - - src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs - - src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs - - src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs -decisions: - - "MySQL InsertJobWithSequence uses derived-table COALESCE form to avoid 'can't specify target table for update in FROM clause' restriction" - - "PostgreSQL InsertJobWithSequence also uses derived-table form for defensive consistency (same table subquery edge case)" - - "SQL Server TOP(batchSize) is C# string-interpolated into the format string (integer literal, not an EF parameter)" - - "SQL Server WITH (UPDLOCK, READPAST, ROWLOCK) applied only to outer FROM clause; CTE bodies have no hints to prevent lock escalation" -metrics: - duration: "~15 minutes" - completed: "2026-05-04" - tasks_completed: 2 - tasks_total: 2 - files_modified: 4 - commits: 2 ---- - -# Phase 09 Plan 02: FIFO SQL Implementation — Dialect Methods and Storage Wiring Summary - -All three SQL dialect classes now implement `InsertJobWithSequence` with atomic per-(queue, partition_key) sequence assignment, and FIFO-aware `GetDueJobs` using `blocked_partitions` + `partition_heads` CTEs. `EntityFrameworkCoreStorage.InsertAsync` routes partitioned jobs through the dialect SQL path and reads back the assigned `SequenceNumber`, with a CR-01 fix assigning the existing `SequenceNumber` on idempotency collision. - -## Tasks Completed - -| Task | Description | Commit | -|------|-------------|--------| -| 1 | InsertJobWithSequence + CTE GetDueJobs in all three dialect classes | bd51d1b | -| 2 | EntityFrameworkCoreStorage.InsertAsync partitioned branch + CR-01 fix | c027831 | - -## What Was Built - -### Task 1 — Dialect SQL Methods (all three providers) - -**`GetDueJobs` replaced with CTE approach:** - -Every dialect now emits a two-CTE query: -1. `blocked_partitions` — collects partition keys that are currently `Processing` OR have been previously attempted (`Pending` with `Attempts > 0`); these are invisible to the poller. -2. `partition_heads` — finds the lowest `SequenceNumber` per unblocked partition. - -The outer `SELECT` then returns: -- Unpartitioned jobs (`PartitionKey IS NULL`) matching the existing eligibility conditions. -- Partitioned jobs where the job is the `partition_head` (head-of-partition only). - -Provider-specific locking is placed exclusively on the outer `FROM` clause: -- PostgreSQL: `FOR NO KEY UPDATE SKIP LOCKED` at end of outer SELECT -- SQL Server: `WITH (UPDLOCK, READPAST, ROWLOCK)` on `FROM {table} AS t` in outer SELECT only; CTE bodies have no hints -- MySQL: `FOR UPDATE SKIP LOCKED` at end of outer SELECT; `partition_heads` CTE uses LEFT JOIN anti-join pattern instead of `NOT IN (subquery on same table)` - -**`InsertJobWithSequence` added to all three dialects:** - -Uses `INSERT INTO ... SELECT ... COALESCE(MAX(SequenceNumber), 0) + 1` pattern, scoped to `(queue_key, partition_key)`. All 17 fields are passed as EF parameterized placeholders — no string concatenation of user values. - -Both MySQL and PostgreSQL use the derived-table form for the `COALESCE(MAX(...))` subquery (`SELECT MAX(...) FROM (SELECT ... FROM {table} WHERE ...) AS sub`) to avoid the "can't specify target table for update in FROM clause" restriction that affects MySQL 5.x and some edge cases in PostgreSQL. - -SQL Server uses the direct subquery form (no derived table needed). - -### Task 2 — EntityFrameworkCoreStorage.InsertAsync - -Two targeted changes: - -**CR-01 fix (idempotency collision):** When an existing job is found with matching `IdempotencyKey`, `job.SequenceNumber = existing.SequenceNumber` is now assigned before returning `existing.Id`. This ensures the caller receives the originally assigned sequence number on a duplicate enqueue. - -**Partitioned insert branch:** After the idempotency check, a new branch checks `job.PartitionKey != null && _providerCache is { IsSupportedProvider: true, Dialect: not null }`. If true, it: -1. Calls `_providerCache.Dialect.InsertJobWithSequence(job)` and executes it via `ExecuteSqlInterpolatedAsync`. -2. Reads back `SequenceNumber` from the database via a separate `FirstAsync` query on `JobEntities`. -3. Assigns `job.SequenceNumber = assigned` and returns `job.Id`. - -The unpartitioned path (`JobEntities.Add(entity); await _dbContext.SaveChangesAsync(...)`) is unchanged. - -## Deviations from Plan - -### Auto-fixed Issues - -**1. [Rule 2 - Security/Correctness] Derived-table form for PostgreSQL InsertJobWithSequence** -- **Found during:** Task 1 implementation -- **Issue:** The plan noted MySQL requires derived-table COALESCE form to avoid same-table restriction; PostgreSQL is generally fine with direct subquery in INSERT...SELECT but the defensive form is safer and the plan explicitly allowed it -- **Fix:** Used `SELECT MAX(seq) FROM (SELECT SequenceNumber FROM {table} WHERE ...) AS sub` for PostgreSQL as well -- **Files modified:** src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs -- **Commit:** bd51d1b - -## Known Stubs - -None — all FIFO SQL implementation paths are fully wired. - -## Threat Flags - -None — all values passed to `FormattableStringFactory.Create` are EF-parameterized. Column/table names come from `EntityMap` (derived from EF model metadata, not user input). `TOP({batchSize})` in SQL Server is an integer literal interpolated at method call time, not a runtime user value. - -## Self-Check: PASSED - -Files exist: -- src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs — FOUND -- src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs — FOUND -- src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs — FOUND -- src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs — FOUND - -Commits exist: -- bd51d1b — FOUND -- c027831 — FOUND - -Build: `dotnet build` — 0 errors - -Acceptance criteria all verified: -- `InsertJobWithSequence` count = 1 in each dialect file -- `blocked_partitions` count >= 1 in each dialect file -- `WITH (UPDLOCK, READPAST, ROWLOCK)` on outer FROM only in SqlServerDialect -- `FOR UPDATE SKIP LOCKED` on outer SELECT only in MySqlDialect -- `FOR NO KEY UPDATE SKIP LOCKED` on outer SELECT only in PostgreSqlDialect -- `InsertJobWithSequence` count = 1 in EntityFrameworkCoreStorage.cs -- `job.SequenceNumber = existing.SequenceNumber` count = 1 (non-comment line) -- `job.SequenceNumber = assigned` count = 1 -- `job.PartitionKey != null` count = 1 From c68a99643910ef4e9dc0ad6221813257eab1c873 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 21:05:03 +0200 Subject: [PATCH 50/53] refactor(storage): simplify job error cleanup and update schedule method names --- .../Providers/ISqlDialect.cs | 2 +- .../Providers/Sql/BaseSqlDialect.cs | 5 +- .../Providers/Sql/MySqlDialect.cs | 40 ++++++------ .../Providers/Sql/PostgreSqlDialect.cs | 40 ++++++------ .../Providers/Sql/SqlServerDialect.cs | 62 +++++++++---------- .../Storage/EntityFrameworkCoreStorage.cs | 29 ++++----- src/Atomizer/Storage/InMemoryStorage.cs | 13 +--- .../Providers/MySqlDialectTests.cs | 4 +- .../Providers/PostgreSqlDialectTests.cs | 4 +- .../Providers/SqlServerDialectTests.cs | 4 +- .../EntityFrameworkCoreStorageTests.cs | 5 +- .../MySql/MySqlStorageContractTests.cs | 6 +- .../Postgres/PostgresStorageContractTests.cs | 6 +- .../SqlServerStorageContractTests.cs | 6 +- .../Sqlite/SqliteStorageContractTests.cs | 16 +---- .../Storage/StorageTestCleanup.cs | 15 +++++ .../Atomizer.Tests.Utilities.csproj | 1 - 17 files changed, 109 insertions(+), 149 deletions(-) create mode 100644 tests/Atomizer.EntityFrameworkCore.Tests/Storage/StorageTestCleanup.cs diff --git a/src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs index e2e991a..0899b30 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/ISqlDialect.cs @@ -5,7 +5,7 @@ internal interface ISqlDialect FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int batchSize); FormattableString ReleaseLeasedJobs(LeaseToken leaseToken, DateTimeOffset now); FormattableString GetDueSchedules(DateTimeOffset now); - FormattableString UpsertScheduleAsync(AtomizerSchedule schedule, DateTimeOffset now); + FormattableString UpsertSchedule(AtomizerSchedule schedule, DateTimeOffset now); /// /// Returns provider-specific SQL that inserts a partitioned job and atomically assigns diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/BaseSqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/BaseSqlDialect.cs index aefe79f..701782b 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/BaseSqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/BaseSqlDialect.cs @@ -90,8 +90,7 @@ protected static string SerializeIntervals(TimeSpan[] intervals) => public FormattableString ReleaseLeasedJobs(LeaseToken leaseToken, DateTimeOffset now) { - var format = - $$""" + var format = $$""" UPDATE {{_jTable}} SET {{_jStatus}} = {{_statusPending}}, {{_jLeaseToken}} = NULL, @@ -106,5 +105,5 @@ public FormattableString ReleaseLeasedJobs(LeaseToken leaseToken, DateTimeOffset public abstract FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int batchSize); public abstract FormattableString InsertJobWithSequence(AtomizerJob job); public abstract FormattableString GetDueSchedules(DateTimeOffset now); - public abstract FormattableString UpsertScheduleAsync(AtomizerSchedule schedule, DateTimeOffset now); + public abstract FormattableString UpsertSchedule(AtomizerSchedule schedule, DateTimeOffset now); } diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs index 9371a0e..23463d2 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs @@ -7,8 +7,7 @@ internal sealed class MySqlDialect(EntityMap jobs, EntityMap schedules) : BaseSq { public override FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int batchSize) { - var format = - $$""" + var format = $$""" WITH blocked_partitions AS ( SELECT DISTINCT {{_jPartitionKey}} FROM {{_jTable}} @@ -65,19 +64,19 @@ LEFT JOIN partition_heads ph """; return FormattableStringFactory.Create( format, - queueKey.Key, // {0} blocked_partitions queue filter - queueKey.Key, // {1} partition_heads queue filter - queueKey.Key, // {2} outer SELECT queue filter - now, // {3} unpartitioned VisibleAt - now, // {4} unpartitioned ScheduledAt - now, // {5} unpartitioned Processing VisibleAt - now, // {6} partitioned VisibleAt - now, // {7} partitioned ScheduledAt - now, // {8} partitioned Processing VisibleAt - batchSize, // {9} LIMIT - now, // {10} partition_heads VisibleAt - now, // {11} partition_heads ScheduledAt - now // {12} partition_heads Processing VisibleAt + queueKey.Key, // {0} blocked_partitions queue filter + queueKey.Key, // {1} partition_heads queue filter + queueKey.Key, // {2} outer SELECT queue filter + now, // {3} unpartitioned VisibleAt + now, // {4} unpartitioned ScheduledAt + now, // {5} unpartitioned Processing VisibleAt + now, // {6} partitioned VisibleAt + now, // {7} partitioned ScheduledAt + now, // {8} partitioned Processing VisibleAt + batchSize, // {9} LIMIT + now, // {10} partition_heads VisibleAt + now, // {11} partition_heads ScheduledAt + now // {12} partition_heads Processing VisibleAt ); } @@ -85,8 +84,7 @@ public override FormattableString InsertJobWithSequence(AtomizerJob job) { var entity = job.ToEntity(); var retryIntervals = SerializeIntervals(entity.RetryIntervals); - var format = - $$""" + var format = $$""" INSERT INTO {{_jTable}} ( {{_jId}}, {{_jQueueKey}}, {{_jPayloadType}}, {{_jPayload}}, {{_jScheduledAt}}, {{_jVisibleAt}}, {{_jStatus}}, {{_jAttempts}}, @@ -125,8 +123,7 @@ public override FormattableString InsertJobWithSequence(AtomizerJob job) public override FormattableString GetDueSchedules(DateTimeOffset now) { - var format = - $$""" + var format = $$""" SELECT t.* FROM {{_sTable}} AS t WHERE {{_sNextRunAt}} <= {0} @@ -137,12 +134,11 @@ SELECT t.* return FormattableStringFactory.Create(format, now); } - public override FormattableString UpsertScheduleAsync(AtomizerSchedule schedule, DateTimeOffset now) + public override FormattableString UpsertSchedule(AtomizerSchedule schedule, DateTimeOffset now) { var entity = schedule.ToEntity(); var retryIntervals = SerializeIntervals(entity.RetryIntervals); - var format = - $$""" + var format = $$""" INSERT INTO {{_sTable}} ( {{_sId}}, {{_sJobKey}}, diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs index 470d449..5c302aa 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs @@ -7,8 +7,7 @@ internal sealed class PostgreSqlDialect(EntityMap jobs, EntityMap schedules) : B { public override FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int batchSize) { - var format = - $$""" + var format = $$""" WITH blocked_partitions AS ( SELECT DISTINCT {{_jPartitionKey}} FROM {{_jTable}} @@ -64,19 +63,19 @@ LEFT JOIN partition_heads ph """; return FormattableStringFactory.Create( format, - queueKey.Key, // {0} blocked_partitions queue filter - queueKey.Key, // {1} partition_heads queue filter - queueKey.Key, // {2} outer SELECT queue filter - now, // {3} unpartitioned VisibleAt - now, // {4} unpartitioned ScheduledAt - now, // {5} unpartitioned Processing VisibleAt - now, // {6} partitioned VisibleAt - now, // {7} partitioned ScheduledAt - now, // {8} partitioned Processing VisibleAt - batchSize, // {9} LIMIT - now, // {10} partition_heads VisibleAt - now, // {11} partition_heads ScheduledAt - now // {12} partition_heads Processing VisibleAt + queueKey.Key, // {0} blocked_partitions queue filter + queueKey.Key, // {1} partition_heads queue filter + queueKey.Key, // {2} outer SELECT queue filter + now, // {3} unpartitioned VisibleAt + now, // {4} unpartitioned ScheduledAt + now, // {5} unpartitioned Processing VisibleAt + now, // {6} partitioned VisibleAt + now, // {7} partitioned ScheduledAt + now, // {8} partitioned Processing VisibleAt + batchSize, // {9} LIMIT + now, // {10} partition_heads VisibleAt + now, // {11} partition_heads ScheduledAt + now // {12} partition_heads Processing VisibleAt ); } @@ -84,8 +83,7 @@ public override FormattableString InsertJobWithSequence(AtomizerJob job) { var entity = job.ToEntity(); var retryIntervals = SerializeIntervals(entity.RetryIntervals); - var format = - $$""" + var format = $$""" INSERT INTO {{_jTable}} ( {{_jId}}, {{_jQueueKey}}, {{_jPayloadType}}, {{_jPayload}}, {{_jScheduledAt}}, {{_jVisibleAt}}, {{_jStatus}}, {{_jAttempts}}, @@ -124,8 +122,7 @@ public override FormattableString InsertJobWithSequence(AtomizerJob job) public override FormattableString GetDueSchedules(DateTimeOffset now) { - var format = - $$""" + var format = $$""" SELECT t.* FROM {{_sTable}} AS t WHERE {{_sEnabled}} = TRUE @@ -136,12 +133,11 @@ SELECT t.* return FormattableStringFactory.Create(format, now); } - public override FormattableString UpsertScheduleAsync(AtomizerSchedule schedule, DateTimeOffset now) + public override FormattableString UpsertSchedule(AtomizerSchedule schedule, DateTimeOffset now) { var entity = schedule.ToEntity(); var retryIntervals = SerializeIntervals(entity.RetryIntervals); - var format = - $$""" + var format = $$""" INSERT INTO {{_sTable}} ( {{_sId}}, {{_sJobKey}}, diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs index 465741c..1032656 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs @@ -7,8 +7,7 @@ internal sealed class SqlServerDialect(EntityMap jobs, EntityMap schedules) : Ba { public override FormattableString GetDueJobs(QueueKey queueKey, DateTimeOffset now, int batchSize) { - var format = - $$""" + var format = $$""" WITH blocked_partitions AS ( SELECT DISTINCT {{_jPartitionKey}} FROM {{_jTable}} @@ -62,18 +61,18 @@ LEFT JOIN partition_heads ph """; return FormattableStringFactory.Create( format, - queueKey.Key, // {0} blocked_partitions queue filter - queueKey.Key, // {1} partition_heads queue filter - queueKey.Key, // {2} outer SELECT queue filter - now, // {3} unpartitioned VisibleAt - now, // {4} unpartitioned ScheduledAt - now, // {5} unpartitioned Processing VisibleAt - now, // {6} partitioned VisibleAt - now, // {7} partitioned ScheduledAt - now, // {8} partitioned Processing VisibleAt - now, // {9} partition_heads VisibleAt (batchSize is TOP(batchSize) inlined, not a placeholder) - now, // {10} partition_heads ScheduledAt - now // {11} partition_heads Processing VisibleAt + queueKey.Key, // {0} blocked_partitions queue filter + queueKey.Key, // {1} partition_heads queue filter + queueKey.Key, // {2} outer SELECT queue filter + now, // {3} unpartitioned VisibleAt + now, // {4} unpartitioned ScheduledAt + now, // {5} unpartitioned Processing VisibleAt + now, // {6} partitioned VisibleAt + now, // {7} partitioned ScheduledAt + now, // {8} partitioned Processing VisibleAt + now, // {9} partition_heads VisibleAt (batchSize is TOP(batchSize) inlined, not a placeholder) + now, // {10} partition_heads ScheduledAt + now // {11} partition_heads Processing VisibleAt ); } @@ -81,8 +80,7 @@ public override FormattableString InsertJobWithSequence(AtomizerJob job) { var entity = job.ToEntity(); var retryIntervals = SerializeIntervals(entity.RetryIntervals); - var format = - $$""" + var format = $$""" INSERT INTO {{_jTable}} ( {{_jId}}, {{_jQueueKey}}, {{_jPayloadType}}, {{_jPayload}}, {{_jScheduledAt}}, {{_jVisibleAt}}, {{_jStatus}}, {{_jAttempts}}, @@ -121,8 +119,7 @@ public override FormattableString InsertJobWithSequence(AtomizerJob job) public override FormattableString GetDueSchedules(DateTimeOffset now) { - var format = - $$""" + var format = $$""" SELECT t.* FROM {{_sTable}} AS t WITH (UPDLOCK, READPAST, ROWLOCK) WHERE {{_sNextRunAt}} <= {0} @@ -132,12 +129,11 @@ SELECT t.* return FormattableStringFactory.Create(format, now); } - public override FormattableString UpsertScheduleAsync(AtomizerSchedule schedule, DateTimeOffset now) + public override FormattableString UpsertSchedule(AtomizerSchedule schedule, DateTimeOffset now) { var entity = schedule.ToEntity(); var retryIntervals = SerializeIntervals(entity.RetryIntervals); - var format = - $$""" + var format = $$""" MERGE {{_sTable}} WITH (HOLDLOCK) AS target USING (SELECT {0}) AS src ({{_sJobKey}}) ON target.{{_sJobKey}} = src.{{_sJobKey}} @@ -189,21 +185,21 @@ WHEN NOT MATCHED THEN INSERT ( """; return FormattableStringFactory.Create( format, - entity.JobKey, // {0} - entity.QueueKey, // {1} - entity.PayloadType, // {2} - entity.Payload, // {3} - entity.Schedule, // {4} - entity.TimeZone, // {5} + entity.JobKey, // {0} + entity.QueueKey, // {1} + entity.PayloadType, // {2} + entity.Payload, // {3} + entity.Schedule, // {4} + entity.TimeZone, // {5} (int)entity.MisfirePolicy, // {6} - entity.MaxCatchUp, // {7} + entity.MaxCatchUp, // {7} entity.Enabled ? 1 : 0, // {8} - retryIntervals, // {9} - entity.NextRunAt, // {10} - now, // {11} - entity.Id, // {12} + retryIntervals, // {9} + entity.NextRunAt, // {10} + now, // {11} + entity.Id, // {12} entity.LastEnqueueAt, // {13} - entity.CreatedAt // {14} + entity.CreatedAt // {14} ); } } diff --git a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs index 0e7edc4..86fc0b4 100644 --- a/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs +++ b/src/Atomizer.EntityFrameworkCore/Storage/EntityFrameworkCoreStorage.cs @@ -180,11 +180,7 @@ CancellationToken cancellationToken .ToList(); } - throw new NotSupportedException( - "The current database provider is not supported. " - + "To bypass this check, set AllowUnsafeProviderFallback to true in EntityFrameworkCoreJobStorageOptions. " - + "Note that this may lead to unexpected behavior." - ); + throw UnsupportedProviderException(); } public async Task ReleaseLeasedAsync( @@ -230,7 +226,7 @@ public async Task UpsertScheduleAsync(AtomizerSchedule schedule, Cancellat if (_providerCache is { IsSupportedProvider: true, Dialect: not null }) { var now = _clock.UtcNow; - var sql = _providerCache.Dialect.UpsertScheduleAsync(schedule, now); + var sql = _providerCache.Dialect.UpsertSchedule(schedule, now); await _dbContext.Database.ExecuteSqlInterpolatedAsync(sql, cancellationToken); return await ScheduleEntities .Where(s => s.JobKey == entity.JobKey) @@ -240,7 +236,6 @@ public async Task UpsertScheduleAsync(AtomizerSchedule schedule, Cancellat if (!_providerCache.IsSupportedProvider && _options.AllowUnsafeProviderFallback) { - // Not race-safe - use only with 1 service running var existing = await ScheduleEntities .AsNoTracking() .FirstOrDefaultAsync(s => s.JobKey == entity.JobKey, cancellationToken); @@ -262,16 +257,13 @@ public async Task UpsertScheduleAsync(AtomizerSchedule schedule, Cancellat catch (DbUpdateException ex) { _logger.LogError(ex, "Failed to upsert schedule for job {JobKey}", schedule.JobKey); + throw; } return entity.Id; } - throw new NotSupportedException( - "The current database provider is not supported. " - + "To bypass this check, set AllowUnsafeProviderFallback to true in EntityFrameworkCoreJobStorageOptions. " - + "Note that this may lead to unexpected behavior." - ); + throw UnsupportedProviderException(); } public async Task UpdateSchedulesAsync(IEnumerable schedules, CancellationToken cancellationToken) @@ -321,11 +313,7 @@ CancellationToken cancellationToken .ToListAsync(cancellationToken); } - throw new NotSupportedException( - "The current database provider is not supported. " - + "To bypass this check, set AllowUnsafeProviderFallback to true in EntityFrameworkCoreJobStorageOptions. " - + "Note that this may lead to unexpected behavior." - ); + throw UnsupportedProviderException(); } public async Task ExecuteInLeaseAsync( @@ -382,4 +370,11 @@ CancellationToken cancellationToken throw; } } + + private static NotSupportedException UnsupportedProviderException() => + new( + "The current database provider is not supported. " + + "To bypass this check, set AllowUnsafeProviderFallback to true in EntityFrameworkCoreJobStorageOptions. " + + "Note that this may lead to unexpected behavior." + ); } diff --git a/src/Atomizer/Storage/InMemoryStorage.cs b/src/Atomizer/Storage/InMemoryStorage.cs index f5f3aea..87286ca 100644 --- a/src/Atomizer/Storage/InMemoryStorage.cs +++ b/src/Atomizer/Storage/InMemoryStorage.cs @@ -40,7 +40,6 @@ public Task InsertAsync(AtomizerJob job, CancellationToken cancellationTok { cancellationToken.ThrowIfCancellationRequested(); - // 1) CR-01 idempotency check — linear scan is acceptable for in-process storage if (job.IdempotencyKey != null) { var existing = _jobs.Values.FirstOrDefault(j => j.IdempotencyKey == job.IdempotencyKey); @@ -51,7 +50,6 @@ public Task InsertAsync(AtomizerJob job, CancellationToken cancellationTok } } - // 2) FIFO-09 sequence assignment — only for partitioned, non-duplicate jobs if (job.PartitionKey != null) { var partitionSequences = _partitionSequences.GetOrAdd( @@ -62,7 +60,6 @@ public Task InsertAsync(AtomizerJob job, CancellationToken cancellationTok job.SequenceNumber = seq; } - // 3) Store + index (unchanged) _jobs[job.Id] = job; IndexIntoQueue(job); @@ -118,17 +115,12 @@ CancellationToken cancellationToken now ); - List candidates; - if (!_queues.TryGetValue(queueKey, out var ids) || ids.IsEmpty) { _logger.LogDebug("LeaseBatch: queue {QueueKey} is empty", queueKey); return Task.FromResult((IReadOnlyList)Array.Empty()); } - // Pass 1: collect blocked partition keys from the FULL queue snapshot - // D-04: blocking check must scan ALL jobs in the queue, not just due-time candidates - // A partition is blocked if any job has IsPartitionBlocked == true (Processing OR Pending+Attempts>0) var blockedPartitions = new HashSet(); foreach (var id in ids.Keys) { @@ -136,7 +128,6 @@ CancellationToken cancellationToken blockedPartitions.Add(bj.PartitionKey!.Key); } - // Pass 2: filter eligible candidates (existing status+time logic) and exclude blocked partitions var eligible = ids .Keys.Select(id => _jobs.TryGetValue(id, out var j) ? j : null) .Where(j => @@ -152,15 +143,13 @@ CancellationToken cancellationToken ) .Select(j => j!); - // Pass 3: FIFO head-of-partition selection - // Use OrderBy().First() not MinBy() — MinBy is .NET 6+ and src/Atomizer targets netstandard2.0 var unpartitioned = eligible.Where(j => j.PartitionKey == null); var partitionHeads = eligible .Where(j => j.PartitionKey != null) .GroupBy(j => j.PartitionKey!.Key) .Select(g => g.OrderBy(j => j.SequenceNumber).First()); - candidates = unpartitioned + var candidates = unpartitioned .Concat(partitionHeads) .OrderBy(j => j.ScheduledAt) .ThenBy(j => j.CreatedAt) diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Providers/MySqlDialectTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Providers/MySqlDialectTests.cs index 00082fe..9aefd83 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Providers/MySqlDialectTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Providers/MySqlDialectTests.cs @@ -59,7 +59,7 @@ public void ReleaseLeasedJobs_WhenCalled_ShouldContainUpdateStatement() } [Fact] - public void UpsertScheduleAsync_WhenCalled_ShouldContainOnDuplicateKeyUpdate() + public void UpsertSchedule_WhenCalled_ShouldContainOnDuplicateKeyUpdate() { var (jobs, schedules) = BuildMaps(); var dialect = new MySqlDialect(jobs, schedules); @@ -73,7 +73,7 @@ public void UpsertScheduleAsync_WhenCalled_ShouldContainOnDuplicateKeyUpdate() DateTimeOffset.UtcNow ); - var sql = dialect.UpsertScheduleAsync(schedule, DateTimeOffset.UtcNow); + var sql = dialect.UpsertSchedule(schedule, DateTimeOffset.UtcNow); sql.Format.Should().Contain("ON DUPLICATE KEY UPDATE"); } diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Providers/PostgreSqlDialectTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Providers/PostgreSqlDialectTests.cs index cf8bf32..35bcd51 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Providers/PostgreSqlDialectTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Providers/PostgreSqlDialectTests.cs @@ -60,7 +60,7 @@ public void ReleaseLeasedJobs_WhenCalled_ShouldContainUpdateStatement() } [Fact] - public void UpsertScheduleAsync_WhenCalled_ShouldContainOnConflict() + public void UpsertSchedule_WhenCalled_ShouldContainOnConflict() { var (jobs, schedules) = BuildMaps(); var dialect = new PostgreSqlDialect(jobs, schedules); @@ -74,7 +74,7 @@ public void UpsertScheduleAsync_WhenCalled_ShouldContainOnConflict() DateTimeOffset.UtcNow ); - var sql = dialect.UpsertScheduleAsync(schedule, DateTimeOffset.UtcNow); + var sql = dialect.UpsertSchedule(schedule, DateTimeOffset.UtcNow); sql.Format.Should().Contain("ON CONFLICT"); sql.Format.Should().Contain("DO UPDATE SET"); diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Providers/SqlServerDialectTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Providers/SqlServerDialectTests.cs index 02fb925..d86c308 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Providers/SqlServerDialectTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Providers/SqlServerDialectTests.cs @@ -60,7 +60,7 @@ public void ReleaseLeasedJobs_WhenCalled_ShouldContainUpdateStatement() } [Fact] - public void UpsertScheduleAsync_WhenCalled_ShouldContainMergeWithHoldlock() + public void UpsertSchedule_WhenCalled_ShouldContainMergeWithHoldlock() { var (jobs, schedules) = BuildMaps(); var dialect = new SqlServerDialect(jobs, schedules); @@ -74,7 +74,7 @@ public void UpsertScheduleAsync_WhenCalled_ShouldContainMergeWithHoldlock() DateTimeOffset.UtcNow ); - var sql = dialect.UpsertScheduleAsync(schedule, DateTimeOffset.UtcNow); + var sql = dialect.UpsertSchedule(schedule, DateTimeOffset.UtcNow); sql.Format.Should().Contain("MERGE"); sql.Format.Should().Contain("WITH (HOLDLOCK)"); diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/EntityFrameworkCoreStorageTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/EntityFrameworkCoreStorageTests.cs index 3e29963..eaaded3 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/EntityFrameworkCoreStorageTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/EntityFrameworkCoreStorageTests.cs @@ -671,10 +671,7 @@ public async Task UpsertScheduleAsync_WhenScheduleExists_ShouldUpdateSchedule() public async ValueTask DisposeAsync() { await using var dbContext = _dbContextFactory(); - dbContext.Set().RemoveRange(dbContext.Set()); - dbContext.Set().RemoveRange(dbContext.Set()); - dbContext.Set().RemoveRange(dbContext.Set()); - await dbContext.SaveChangesAsync(TestContext.Current.CancellationToken); + await StorageTestCleanup.ClearAsync(dbContext, TestContext.Current.CancellationToken); } public ValueTask InitializeAsync() diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs index c84abdb..685edb5 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/MySql/MySqlStorageContractTests.cs @@ -1,6 +1,5 @@ using Atomizer.Abstractions; using Atomizer.Core; -using Atomizer.EntityFrameworkCore.Entities; using Atomizer.EntityFrameworkCore.Storage; using Atomizer.EntityFrameworkCore.Tests.Fixtures; using Atomizer.EntityFrameworkCore.Tests.TestSetup.MySql; @@ -35,9 +34,6 @@ public override async ValueTask DisposeAsync() // Use a bounded cancellation token so teardown does not hang indefinitely. using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); await using var cleanupContext = fixture.CreateNewDbContext(); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); - await cleanupContext.SaveChangesAsync(cts.Token); + await StorageTestCleanup.ClearAsync(cleanupContext, cts.Token); } } diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs index 7b60af1..77104db 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Postgres/PostgresStorageContractTests.cs @@ -1,6 +1,5 @@ using Atomizer.Abstractions; using Atomizer.Core; -using Atomizer.EntityFrameworkCore.Entities; using Atomizer.EntityFrameworkCore.Storage; using Atomizer.EntityFrameworkCore.Tests.Fixtures; using Atomizer.EntityFrameworkCore.Tests.TestSetup.Postgres; @@ -35,9 +34,6 @@ public override async ValueTask DisposeAsync() // Use a bounded cancellation token so teardown does not hang indefinitely. using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); await using var cleanupContext = fixture.CreateNewDbContext(); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); - await cleanupContext.SaveChangesAsync(cts.Token); + await StorageTestCleanup.ClearAsync(cleanupContext, cts.Token); } } diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs index 31f73de..072c976 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/SqlServer/SqlServerStorageContractTests.cs @@ -1,6 +1,5 @@ using Atomizer.Abstractions; using Atomizer.Core; -using Atomizer.EntityFrameworkCore.Entities; using Atomizer.EntityFrameworkCore.Storage; using Atomizer.EntityFrameworkCore.Tests.Fixtures; using Atomizer.EntityFrameworkCore.Tests.TestSetup.SqlServer; @@ -35,9 +34,6 @@ public override async ValueTask DisposeAsync() // Use a bounded cancellation token so teardown does not hang indefinitely. using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); await using var cleanupContext = fixture.CreateNewDbContext(); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); - await cleanupContext.SaveChangesAsync(cts.Token); + await StorageTestCleanup.ClearAsync(cleanupContext, cts.Token); } } diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Sqlite/SqliteStorageContractTests.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Sqlite/SqliteStorageContractTests.cs index f75939c..c54944e 100644 --- a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Sqlite/SqliteStorageContractTests.cs +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/Sqlite/SqliteStorageContractTests.cs @@ -1,6 +1,5 @@ using Atomizer.Abstractions; using Atomizer.Core; -using Atomizer.EntityFrameworkCore.Entities; using Atomizer.EntityFrameworkCore.Storage; using Atomizer.EntityFrameworkCore.Tests.Fixtures; using Atomizer.EntityFrameworkCore.Tests.TestSetup.Sqlite; @@ -17,14 +16,8 @@ namespace Atomizer.EntityFrameworkCore.Tests.Storage.Sqlite; /// The CTE dialect SQL is verified by the PostgreSQL, SQL Server, and MySQL subclasses. /// /// -/// NOTE: The two FIFO-08 partition-blocking tests -/// (GetDueJobsAsync_WhenPartitionIsBlockedByProcessing_ShouldExcludeEntirePartition and -/// GetDueJobsAsync_WhenPartitionIsBlockedByPendingWithAttempts_ShouldExcludeEntirePartition) -/// are expected to FAIL for SQLite. The LINQ fallback path in GetDueJobsAsync does not -/// enforce FIFO partition blocking — it returns all due jobs without partition exclusion. This is a -/// known limitation of the LINQ fallback; FIFO enforcement requires the provider-specific CTE SQL -/// implemented in PostgreSqlDialect, SqlServerDialect, and MySqlDialect. The real providers are -/// the authoritative FIFO test surface. +/// The fallback path enforces FIFO semantics in-process, but it is not safe for concurrent +/// multi-node production use because it does not take provider-level row locks. /// [Collection(nameof(SqliteDatabaseFixture))] public sealed class SqliteStorageContractTests(SqliteDatabaseFixture fixture) : AtomizerStorageContractTests @@ -51,9 +44,6 @@ public override async ValueTask DisposeAsync() // Use a bounded cancellation token so teardown does not hang indefinitely. using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); await using var cleanupContext = fixture.CreateNewDbContext(); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); - cleanupContext.Set().RemoveRange(cleanupContext.Set()); - await cleanupContext.SaveChangesAsync(cts.Token); + await StorageTestCleanup.ClearAsync(cleanupContext, cts.Token); } } diff --git a/tests/Atomizer.EntityFrameworkCore.Tests/Storage/StorageTestCleanup.cs b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/StorageTestCleanup.cs new file mode 100644 index 0000000..390a6e4 --- /dev/null +++ b/tests/Atomizer.EntityFrameworkCore.Tests/Storage/StorageTestCleanup.cs @@ -0,0 +1,15 @@ +using Atomizer.EntityFrameworkCore.Entities; +using Microsoft.EntityFrameworkCore; + +namespace Atomizer.EntityFrameworkCore.Tests.Storage; + +internal static class StorageTestCleanup +{ + public static async Task ClearAsync(DbContext dbContext, CancellationToken cancellationToken) + { + dbContext.Set().RemoveRange(dbContext.Set()); + dbContext.Set().RemoveRange(dbContext.Set()); + dbContext.Set().RemoveRange(dbContext.Set()); + await dbContext.SaveChangesAsync(cancellationToken); + } +} diff --git a/tests/Atomizer.Tests.Utilities/Atomizer.Tests.Utilities.csproj b/tests/Atomizer.Tests.Utilities/Atomizer.Tests.Utilities.csproj index a5bdd91..d828433 100644 --- a/tests/Atomizer.Tests.Utilities/Atomizer.Tests.Utilities.csproj +++ b/tests/Atomizer.Tests.Utilities/Atomizer.Tests.Utilities.csproj @@ -5,7 +5,6 @@ false true 12 - true From 30fc0be491d6b10adf842ccfa695b4fd84438c1a Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 21:25:50 +0200 Subject: [PATCH 51/53] fix: address Codex P1 review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CR-01: Undo Attempt() increment before Release() on cancellation — a cancelled job is not a failed attempt, and Pending+Attempts>0 was causing the partition to be treated as blocked indefinitely. CR-02: Add row-locking to the MAX(sequence)+1 subquery in all three SQL dialects to prevent duplicate sequence numbers under concurrent partitioned inserts. Uses WITH (UPDLOCK, HOLDLOCK) for SQL Server, FOR NO KEY UPDATE for PostgreSQL, and FOR UPDATE for MySQL. Co-Authored-By: Claude Sonnet 4.6 --- .../Providers/Sql/MySqlDialect.cs | 2 +- .../Providers/Sql/PostgreSqlDialect.cs | 2 +- .../Providers/Sql/SqlServerDialect.cs | 2 +- src/Atomizer/Processing/JobProcessor.cs | 4 +++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs index 23463d2..28ee4f9 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/MySqlDialect.cs @@ -97,7 +97,7 @@ public override FormattableString InsertJobWithSequence(AtomizerJob job) {8}, {9}, {10}, {11}, {12}, {13}, {14}, - COALESCE((SELECT MAX(max_seq) FROM (SELECT MAX({{_jSequenceNumber}}) AS max_seq FROM {{_jTable}} WHERE {{_jQueueKey}} = {15} AND {{_jPartitionKey}} = {16}) AS sub), 0) + 1; + COALESCE((SELECT MAX(max_seq) FROM (SELECT MAX({{_jSequenceNumber}}) AS max_seq FROM {{_jTable}} WHERE {{_jQueueKey}} = {15} AND {{_jPartitionKey}} = {16} FOR UPDATE) AS sub), 0) + 1; """; return FormattableStringFactory.Create( format, diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs index 5c302aa..ab7062f 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/PostgreSqlDialect.cs @@ -96,7 +96,7 @@ public override FormattableString InsertJobWithSequence(AtomizerJob job) {8}, {9}, {10}, {11}, {12}, {13}, {14}, - COALESCE((SELECT MAX({{_jSequenceNumber}}) FROM (SELECT {{_jSequenceNumber}} FROM {{_jTable}} WHERE {{_jQueueKey}} = {15} AND {{_jPartitionKey}} = {16}) AS sub), 0) + 1; + COALESCE((SELECT MAX({{_jSequenceNumber}}) FROM (SELECT {{_jSequenceNumber}} FROM {{_jTable}} WHERE {{_jQueueKey}} = {15} AND {{_jPartitionKey}} = {16} FOR NO KEY UPDATE) AS sub), 0) + 1; """; return FormattableStringFactory.Create( format, diff --git a/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs b/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs index 1032656..b0ee014 100644 --- a/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs +++ b/src/Atomizer.EntityFrameworkCore/Providers/Sql/SqlServerDialect.cs @@ -93,7 +93,7 @@ public override FormattableString InsertJobWithSequence(AtomizerJob job) {8}, {9}, {10}, {11}, {12}, {13}, {14}, - COALESCE((SELECT MAX({{_jSequenceNumber}}) FROM {{_jTable}} WHERE {{_jQueueKey}} = {15} AND {{_jPartitionKey}} = {16}), 0) + 1; + COALESCE((SELECT MAX({{_jSequenceNumber}}) FROM {{_jTable}} WITH (UPDLOCK, HOLDLOCK) WHERE {{_jQueueKey}} = {15} AND {{_jPartitionKey}} = {16}), 0) + 1; """; return FormattableStringFactory.Create( format, diff --git a/src/Atomizer/Processing/JobProcessor.cs b/src/Atomizer/Processing/JobProcessor.cs index b6b6b7f..b4d9fb7 100644 --- a/src/Atomizer/Processing/JobProcessor.cs +++ b/src/Atomizer/Processing/JobProcessor.cs @@ -64,7 +64,9 @@ public async Task ProcessAsync(AtomizerJob job, CancellationToken ct) { _logger.LogWarning("Operation cancelled while processing job {JobId} on '{Queue}'", job.Id, job.QueueKey); - // Release the job so its partition (if any) is not blocked for the full visibility timeout. + // Undo the Attempt() increment before releasing: cancellation is not a failed attempt, + // so Pending+Attempts>0 must not leave the partition permanently blocked. + job.Attempts -= 1; job.Release(_clock.UtcNow); using var scope = _serviceScopeFactory.CreateScope(); await scope.Storage.UpdateJobsAsync(new[] { job }, CancellationToken.None); From aff713d7968ad7d53fb80981e5c9c86cf5b73e89 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 21:36:32 +0200 Subject: [PATCH 52/53] chore(samples): regenerate Initial migrations Deleted all existing migrations and regenerated from scratch to reflect the current entity model (FIFO PartitionKey + SequenceNumber columns). Co-Authored-By: Claude Sonnet 4.6 --- .../20250827145643_Initial.Designer.cs | 240 ------------------ .../Migrations/20250827145643_Initial.cs | 176 ------------- ...93358_AddIndexForJobKeyOnSchedulesTable.cs | 76 ------ ....cs => 20260504193620_Initial.Designer.cs} | 19 +- .../Migrations/20260504193620_Initial.cs | 161 ++++++++++++ .../ExampleMySqlContextModelSnapshot.cs | 15 +- .../20250827145628_Initial.Designer.cs | 240 ------------------ ...93345_AddIndexForJobKeyOnSchedulesTable.cs | 74 ------ ....cs => 20260504193605_Initial.Designer.cs} | 19 +- ...8_Initial.cs => 20260504193605_Initial.cs} | 109 +++----- .../ExamplePostgresContextModelSnapshot.cs | 15 +- .../20250827145633_Initial.Designer.cs | 240 ------------------ ...93351_AddIndexForJobKeyOnSchedulesTable.cs | 74 ------ ....cs => 20260504193610_Initial.Designer.cs} | 19 +- ...3_Initial.cs => 20260504193610_Initial.cs} | 59 +++-- .../ExampleSqlServerContextModelSnapshot.cs | 15 +- .../20250827145638_Initial.Designer.cs | 235 ----------------- ...93354_AddIndexForJobKeyOnSchedulesTable.cs | 30 --- ....cs => 20260504193615_Initial.Designer.cs} | 15 +- ...8_Initial.cs => 20260504193615_Initial.cs} | 59 +++-- .../ExampleSqliteContextModelSnapshot.cs | 11 +- 21 files changed, 365 insertions(+), 1536 deletions(-) delete mode 100644 samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20250827145643_Initial.Designer.cs delete mode 100644 samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20250827145643_Initial.cs delete mode 100644 samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260503193358_AddIndexForJobKeyOnSchedulesTable.cs rename samples/Atomizer.EFCore.Example/Data/MySql/Migrations/{20260503193358_AddIndexForJobKeyOnSchedulesTable.Designer.cs => 20260504193620_Initial.Designer.cs} (94%) create mode 100644 samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260504193620_Initial.cs delete mode 100644 samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20250827145628_Initial.Designer.cs delete mode 100644 samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20260503193345_AddIndexForJobKeyOnSchedulesTable.cs rename samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/{20260503193345_AddIndexForJobKeyOnSchedulesTable.Designer.cs => 20260504193605_Initial.Designer.cs} (94%) rename samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/{20250827145628_Initial.cs => 20260504193605_Initial.cs} (65%) delete mode 100644 samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20250827145633_Initial.Designer.cs delete mode 100644 samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20260503193351_AddIndexForJobKeyOnSchedulesTable.cs rename samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/{20260503193351_AddIndexForJobKeyOnSchedulesTable.Designer.cs => 20260504193610_Initial.Designer.cs} (94%) rename samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/{20250827145633_Initial.cs => 20260504193610_Initial.cs} (81%) delete mode 100644 samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20250827145638_Initial.Designer.cs delete mode 100644 samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20260503193354_AddIndexForJobKeyOnSchedulesTable.cs rename samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/{20260503193354_AddIndexForJobKeyOnSchedulesTable.Designer.cs => 20260504193615_Initial.Designer.cs} (95%) rename samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/{20250827145638_Initial.cs => 20260504193615_Initial.cs} (81%) diff --git a/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20250827145643_Initial.Designer.cs b/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20250827145643_Initial.Designer.cs deleted file mode 100644 index ea93656..0000000 --- a/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20250827145643_Initial.Designer.cs +++ /dev/null @@ -1,240 +0,0 @@ -// -using System; -using Atomizer.EFCore.Example.Data.MySql; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Metadata; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; - -#nullable disable - -namespace Atomizer.EFCore.Example.Data.MySql.Migrations -{ - [DbContext(typeof(ExampleMySqlContext))] - [Migration("20250827145643_Initial")] - partial class Initial - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "9.0.8") - .HasAnnotation("Relational:MaxIdentifierLength", 64); - - MySqlModelBuilderExtensions.AutoIncrementColumns(modelBuilder); - - modelBuilder.Entity("Atomizer.EFCore.Example.Entities.Product", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("char(36)"); - - b.Property("CreatedAt") - .HasColumnType("datetime(6)"); - - b.Property("Name") - .IsRequired() - .HasColumnType("longtext"); - - b.Property("Price") - .HasColumnType("decimal(65,30)"); - - b.Property("Quantity") - .HasColumnType("int"); - - b.HasKey("Id"); - - b.ToTable("Products"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("char(36)"); - - b.Property("Attempts") - .HasColumnType("int"); - - b.Property("CompletedAt") - .HasColumnType("datetime(6)"); - - b.Property("CreatedAt") - .HasColumnType("datetime(6)"); - - b.Property("FailedAt") - .HasColumnType("datetime(6)"); - - b.Property("IdempotencyKey") - .HasMaxLength(512) - .HasColumnType("varchar(512)"); - - b.Property("LeaseToken") - .HasMaxLength(512) - .HasColumnType("varchar(512)"); - - b.Property("Payload") - .IsRequired() - .HasColumnType("longtext"); - - b.Property("PayloadType") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("varchar(1024)"); - - b.Property("QueueKey") - .IsRequired() - .HasMaxLength(512) - .HasColumnType("varchar(512)"); - - b.Property("RetryIntervals") - .IsRequired() - .HasMaxLength(4096) - .HasColumnType("varchar(4096)"); - - b.Property("ScheduleJobKey") - .HasMaxLength(512) - .HasColumnType("varchar(512)"); - - b.Property("ScheduledAt") - .HasColumnType("datetime(6)"); - - b.Property("Status") - .HasColumnType("int"); - - b.Property("UpdatedAt") - .HasColumnType("datetime(6)"); - - b.Property("VisibleAt") - .HasColumnType("datetime(6)"); - - b.HasKey("Id"); - - b.ToTable("AtomizerJobs", (string)null); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobErrorEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("char(36)"); - - b.Property("Attempt") - .HasColumnType("int"); - - b.Property("CreatedAt") - .HasColumnType("datetime(6)"); - - b.Property("ErrorMessage") - .HasMaxLength(2048) - .HasColumnType("varchar(2048)"); - - b.Property("ExceptionType") - .HasMaxLength(1024) - .HasColumnType("varchar(1024)"); - - b.Property("JobId") - .HasColumnType("char(36)"); - - b.Property("RuntimeIdentity") - .HasMaxLength(255) - .HasColumnType("varchar(255)"); - - b.Property("StackTrace") - .HasMaxLength(5120) - .HasColumnType("varchar(5120)"); - - b.HasKey("Id"); - - b.HasIndex("JobId"); - - b.ToTable("AtomizerJobErrors", (string)null); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerScheduleEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("char(36)"); - - b.Property("CreatedAt") - .HasColumnType("datetime(6)"); - - b.Property("Enabled") - .HasColumnType("tinyint(1)"); - - b.Property("JobKey") - .IsRequired() - .HasMaxLength(512) - .HasColumnType("varchar(512)"); - - b.Property("LastEnqueueAt") - .HasColumnType("datetime(6)"); - - b.Property("MaxCatchUp") - .HasColumnType("int"); - - b.Property("MisfirePolicy") - .HasColumnType("int"); - - b.Property("NextRunAt") - .HasColumnType("datetime(6)"); - - b.Property("Payload") - .IsRequired() - .HasColumnType("longtext"); - - b.Property("PayloadType") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("varchar(1024)"); - - b.Property("QueueKey") - .IsRequired() - .HasMaxLength(512) - .HasColumnType("varchar(512)"); - - b.Property("RetryIntervals") - .IsRequired() - .HasMaxLength(4096) - .HasColumnType("varchar(4096)"); - - b.Property("Schedule") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("varchar(1024)"); - - b.Property("TimeZone") - .IsRequired() - .HasMaxLength(64) - .HasColumnType("varchar(64)"); - - b.Property("UpdatedAt") - .HasColumnType("datetime(6)"); - - b.HasKey("Id"); - - b.ToTable("AtomizerSchedules", (string)null); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobErrorEntity", b => - { - b.HasOne("Atomizer.EntityFrameworkCore.Entities.AtomizerJobEntity", "Job") - .WithMany("Errors") - .HasForeignKey("JobId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.Navigation("Job"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobEntity", b => - { - b.Navigation("Errors"); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20250827145643_Initial.cs b/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20250827145643_Initial.cs deleted file mode 100644 index c768c0d..0000000 --- a/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20250827145643_Initial.cs +++ /dev/null @@ -1,176 +0,0 @@ -using System; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace Atomizer.EFCore.Example.Data.MySql.Migrations -{ - /// - public partial class Initial : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.AlterDatabase().Annotation("MySql:CharSet", "utf8mb4"); - - migrationBuilder - .CreateTable( - name: "AtomizerJobs", - columns: table => new - { - Id = table.Column(type: "char(36)", nullable: false, collation: "ascii_general_ci"), - QueueKey = table - .Column(type: "varchar(512)", maxLength: 512, nullable: false) - .Annotation("MySql:CharSet", "utf8mb4"), - PayloadType = table - .Column(type: "varchar(1024)", maxLength: 1024, nullable: false) - .Annotation("MySql:CharSet", "utf8mb4"), - Payload = table - .Column(type: "longtext", nullable: false) - .Annotation("MySql:CharSet", "utf8mb4"), - ScheduledAt = table.Column(type: "datetime(6)", nullable: false), - VisibleAt = table.Column(type: "datetime(6)", nullable: true), - Status = table.Column(type: "int", nullable: false), - Attempts = table.Column(type: "int", nullable: false), - RetryIntervals = table - .Column(type: "varchar(4096)", maxLength: 4096, nullable: false) - .Annotation("MySql:CharSet", "utf8mb4"), - CreatedAt = table.Column(type: "datetime(6)", nullable: false), - UpdatedAt = table.Column(type: "datetime(6)", nullable: false), - CompletedAt = table.Column(type: "datetime(6)", nullable: true), - FailedAt = table.Column(type: "datetime(6)", nullable: true), - LeaseToken = table - .Column(type: "varchar(512)", maxLength: 512, nullable: true) - .Annotation("MySql:CharSet", "utf8mb4"), - ScheduleJobKey = table - .Column(type: "varchar(512)", maxLength: 512, nullable: true) - .Annotation("MySql:CharSet", "utf8mb4"), - IdempotencyKey = table - .Column(type: "varchar(512)", maxLength: 512, nullable: true) - .Annotation("MySql:CharSet", "utf8mb4"), - }, - constraints: table => - { - table.PrimaryKey("PK_AtomizerJobs", x => x.Id); - } - ) - .Annotation("MySql:CharSet", "utf8mb4"); - - migrationBuilder - .CreateTable( - name: "AtomizerSchedules", - columns: table => new - { - Id = table.Column(type: "char(36)", nullable: false, collation: "ascii_general_ci"), - JobKey = table - .Column(type: "varchar(512)", maxLength: 512, nullable: false) - .Annotation("MySql:CharSet", "utf8mb4"), - QueueKey = table - .Column(type: "varchar(512)", maxLength: 512, nullable: false) - .Annotation("MySql:CharSet", "utf8mb4"), - PayloadType = table - .Column(type: "varchar(1024)", maxLength: 1024, nullable: false) - .Annotation("MySql:CharSet", "utf8mb4"), - Payload = table - .Column(type: "longtext", nullable: false) - .Annotation("MySql:CharSet", "utf8mb4"), - Schedule = table - .Column(type: "varchar(1024)", maxLength: 1024, nullable: false) - .Annotation("MySql:CharSet", "utf8mb4"), - TimeZone = table - .Column(type: "varchar(64)", maxLength: 64, nullable: false) - .Annotation("MySql:CharSet", "utf8mb4"), - MisfirePolicy = table.Column(type: "int", nullable: false), - MaxCatchUp = table.Column(type: "int", nullable: false), - Enabled = table.Column(type: "tinyint(1)", nullable: false), - RetryIntervals = table - .Column(type: "varchar(4096)", maxLength: 4096, nullable: false) - .Annotation("MySql:CharSet", "utf8mb4"), - NextRunAt = table.Column(type: "datetime(6)", nullable: false), - LastEnqueueAt = table.Column(type: "datetime(6)", nullable: true), - CreatedAt = table.Column(type: "datetime(6)", nullable: false), - UpdatedAt = table.Column(type: "datetime(6)", nullable: false), - }, - constraints: table => - { - table.PrimaryKey("PK_AtomizerSchedules", x => x.Id); - } - ) - .Annotation("MySql:CharSet", "utf8mb4"); - - migrationBuilder - .CreateTable( - name: "Products", - columns: table => new - { - Id = table.Column(type: "char(36)", nullable: false, collation: "ascii_general_ci"), - Name = table - .Column(type: "longtext", nullable: false) - .Annotation("MySql:CharSet", "utf8mb4"), - Price = table.Column(type: "decimal(65,30)", nullable: false), - CreatedAt = table.Column(type: "datetime(6)", nullable: false), - Quantity = table.Column(type: "int", nullable: false), - }, - constraints: table => - { - table.PrimaryKey("PK_Products", x => x.Id); - } - ) - .Annotation("MySql:CharSet", "utf8mb4"); - - migrationBuilder - .CreateTable( - name: "AtomizerJobErrors", - columns: table => new - { - Id = table.Column(type: "char(36)", nullable: false, collation: "ascii_general_ci"), - JobId = table.Column(type: "char(36)", nullable: false, collation: "ascii_general_ci"), - ErrorMessage = table - .Column(type: "varchar(2048)", maxLength: 2048, nullable: true) - .Annotation("MySql:CharSet", "utf8mb4"), - StackTrace = table - .Column(type: "varchar(5120)", maxLength: 5120, nullable: true) - .Annotation("MySql:CharSet", "utf8mb4"), - ExceptionType = table - .Column(type: "varchar(1024)", maxLength: 1024, nullable: true) - .Annotation("MySql:CharSet", "utf8mb4"), - CreatedAt = table.Column(type: "datetime(6)", nullable: false), - Attempt = table.Column(type: "int", nullable: false), - RuntimeIdentity = table - .Column(type: "varchar(255)", maxLength: 255, nullable: true) - .Annotation("MySql:CharSet", "utf8mb4"), - }, - constraints: table => - { - table.PrimaryKey("PK_AtomizerJobErrors", x => x.Id); - table.ForeignKey( - name: "FK_AtomizerJobErrors_AtomizerJobs_JobId", - column: x => x.JobId, - principalTable: "AtomizerJobs", - principalColumn: "Id", - onDelete: ReferentialAction.Cascade - ); - } - ) - .Annotation("MySql:CharSet", "utf8mb4"); - - migrationBuilder.CreateIndex( - name: "IX_AtomizerJobErrors_JobId", - table: "AtomizerJobErrors", - column: "JobId" - ); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropTable(name: "AtomizerJobErrors"); - - migrationBuilder.DropTable(name: "AtomizerSchedules"); - - migrationBuilder.DropTable(name: "Products"); - - migrationBuilder.DropTable(name: "AtomizerJobs"); - } - } -} diff --git a/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260503193358_AddIndexForJobKeyOnSchedulesTable.cs b/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260503193358_AddIndexForJobKeyOnSchedulesTable.cs deleted file mode 100644 index cd10b04..0000000 --- a/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260503193358_AddIndexForJobKeyOnSchedulesTable.cs +++ /dev/null @@ -1,76 +0,0 @@ -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace Atomizer.EFCore.Example.Data.MySql.Migrations -{ - /// - public partial class AddIndexForJobKeyOnSchedulesTable : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.AlterColumn( - name: "QueueKey", - table: "AtomizerSchedules", - type: "varchar(100)", - maxLength: 100, - nullable: false, - oldClrType: typeof(string), - oldType: "varchar(512)", - oldMaxLength: 512) - .Annotation("MySql:CharSet", "utf8mb4") - .OldAnnotation("MySql:CharSet", "utf8mb4"); - - migrationBuilder.AlterColumn( - name: "JobKey", - table: "AtomizerSchedules", - type: "varchar(255)", - maxLength: 255, - nullable: false, - oldClrType: typeof(string), - oldType: "varchar(512)", - oldMaxLength: 512) - .Annotation("MySql:CharSet", "utf8mb4") - .OldAnnotation("MySql:CharSet", "utf8mb4"); - - migrationBuilder.CreateIndex( - name: "IX_AtomizerSchedules_JobKey", - table: "AtomizerSchedules", - column: "JobKey", - unique: true); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropIndex( - name: "IX_AtomizerSchedules_JobKey", - table: "AtomizerSchedules"); - - migrationBuilder.AlterColumn( - name: "QueueKey", - table: "AtomizerSchedules", - type: "varchar(512)", - maxLength: 512, - nullable: false, - oldClrType: typeof(string), - oldType: "varchar(100)", - oldMaxLength: 100) - .Annotation("MySql:CharSet", "utf8mb4") - .OldAnnotation("MySql:CharSet", "utf8mb4"); - - migrationBuilder.AlterColumn( - name: "JobKey", - table: "AtomizerSchedules", - type: "varchar(512)", - maxLength: 512, - nullable: false, - oldClrType: typeof(string), - oldType: "varchar(255)", - oldMaxLength: 255) - .Annotation("MySql:CharSet", "utf8mb4") - .OldAnnotation("MySql:CharSet", "utf8mb4"); - } - } -} diff --git a/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260503193358_AddIndexForJobKeyOnSchedulesTable.Designer.cs b/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260504193620_Initial.Designer.cs similarity index 94% rename from samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260503193358_AddIndexForJobKeyOnSchedulesTable.Designer.cs rename to samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260504193620_Initial.Designer.cs index 24c4a77..1d93d78 100644 --- a/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260503193358_AddIndexForJobKeyOnSchedulesTable.Designer.cs +++ b/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260504193620_Initial.Designer.cs @@ -12,8 +12,8 @@ namespace Atomizer.EFCore.Example.Data.MySql.Migrations { [DbContext(typeof(ExampleMySqlContext))] - [Migration("20260503193358_AddIndexForJobKeyOnSchedulesTable")] - partial class AddIndexForJobKeyOnSchedulesTable + [Migration("20260504193620_Initial")] + partial class Initial { /// protected override void BuildTargetModel(ModelBuilder modelBuilder) @@ -75,6 +75,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasMaxLength(512) .HasColumnType("varchar(512)"); + b.Property("PartitionKey") + .HasMaxLength(255) + .HasColumnType("varchar(255)"); + b.Property("Payload") .IsRequired() .HasColumnType("longtext"); @@ -86,8 +90,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("QueueKey") .IsRequired() - .HasMaxLength(512) - .HasColumnType("varchar(512)"); + .HasMaxLength(100) + .HasColumnType("varchar(100)"); b.Property("RetryIntervals") .IsRequired() @@ -95,12 +99,15 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("varchar(4096)"); b.Property("ScheduleJobKey") - .HasMaxLength(512) - .HasColumnType("varchar(512)"); + .HasMaxLength(255) + .HasColumnType("varchar(255)"); b.Property("ScheduledAt") .HasColumnType("datetime(6)"); + b.Property("SequenceNumber") + .HasColumnType("bigint"); + b.Property("Status") .HasColumnType("int"); diff --git a/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260504193620_Initial.cs b/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260504193620_Initial.cs new file mode 100644 index 0000000..14ccf33 --- /dev/null +++ b/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/20260504193620_Initial.cs @@ -0,0 +1,161 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Atomizer.EFCore.Example.Data.MySql.Migrations +{ + /// + public partial class Initial : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterDatabase() + .Annotation("MySql:CharSet", "utf8mb4"); + + migrationBuilder.CreateTable( + name: "AtomizerJobs", + columns: table => new + { + Id = table.Column(type: "char(36)", nullable: false, collation: "ascii_general_ci"), + QueueKey = table.Column(type: "varchar(100)", maxLength: 100, nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + PayloadType = table.Column(type: "varchar(1024)", maxLength: 1024, nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + Payload = table.Column(type: "longtext", nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + ScheduledAt = table.Column(type: "datetime(6)", nullable: false), + VisibleAt = table.Column(type: "datetime(6)", nullable: true), + Status = table.Column(type: "int", nullable: false), + Attempts = table.Column(type: "int", nullable: false), + RetryIntervals = table.Column(type: "varchar(4096)", maxLength: 4096, nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + CreatedAt = table.Column(type: "datetime(6)", nullable: false), + UpdatedAt = table.Column(type: "datetime(6)", nullable: false), + CompletedAt = table.Column(type: "datetime(6)", nullable: true), + FailedAt = table.Column(type: "datetime(6)", nullable: true), + LeaseToken = table.Column(type: "varchar(512)", maxLength: 512, nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + ScheduleJobKey = table.Column(type: "varchar(255)", maxLength: 255, nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + IdempotencyKey = table.Column(type: "varchar(512)", maxLength: 512, nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + PartitionKey = table.Column(type: "varchar(255)", maxLength: 255, nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + SequenceNumber = table.Column(type: "bigint", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_AtomizerJobs", x => x.Id); + }) + .Annotation("MySql:CharSet", "utf8mb4"); + + migrationBuilder.CreateTable( + name: "AtomizerSchedules", + columns: table => new + { + Id = table.Column(type: "char(36)", nullable: false, collation: "ascii_general_ci"), + JobKey = table.Column(type: "varchar(255)", maxLength: 255, nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + QueueKey = table.Column(type: "varchar(100)", maxLength: 100, nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + PayloadType = table.Column(type: "varchar(1024)", maxLength: 1024, nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + Payload = table.Column(type: "longtext", nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + Schedule = table.Column(type: "varchar(1024)", maxLength: 1024, nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + TimeZone = table.Column(type: "varchar(64)", maxLength: 64, nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + MisfirePolicy = table.Column(type: "int", nullable: false), + MaxCatchUp = table.Column(type: "int", nullable: false), + Enabled = table.Column(type: "tinyint(1)", nullable: false), + RetryIntervals = table.Column(type: "varchar(4096)", maxLength: 4096, nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + NextRunAt = table.Column(type: "datetime(6)", nullable: false), + LastEnqueueAt = table.Column(type: "datetime(6)", nullable: true), + CreatedAt = table.Column(type: "datetime(6)", nullable: false), + UpdatedAt = table.Column(type: "datetime(6)", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_AtomizerSchedules", x => x.Id); + }) + .Annotation("MySql:CharSet", "utf8mb4"); + + migrationBuilder.CreateTable( + name: "Products", + columns: table => new + { + Id = table.Column(type: "char(36)", nullable: false, collation: "ascii_general_ci"), + Name = table.Column(type: "longtext", nullable: false) + .Annotation("MySql:CharSet", "utf8mb4"), + Price = table.Column(type: "decimal(65,30)", nullable: false), + CreatedAt = table.Column(type: "datetime(6)", nullable: false), + Quantity = table.Column(type: "int", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Products", x => x.Id); + }) + .Annotation("MySql:CharSet", "utf8mb4"); + + migrationBuilder.CreateTable( + name: "AtomizerJobErrors", + columns: table => new + { + Id = table.Column(type: "char(36)", nullable: false, collation: "ascii_general_ci"), + JobId = table.Column(type: "char(36)", nullable: false, collation: "ascii_general_ci"), + ErrorMessage = table.Column(type: "varchar(2048)", maxLength: 2048, nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + StackTrace = table.Column(type: "varchar(5120)", maxLength: 5120, nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + ExceptionType = table.Column(type: "varchar(1024)", maxLength: 1024, nullable: true) + .Annotation("MySql:CharSet", "utf8mb4"), + CreatedAt = table.Column(type: "datetime(6)", nullable: false), + Attempt = table.Column(type: "int", nullable: false), + RuntimeIdentity = table.Column(type: "varchar(255)", maxLength: 255, nullable: true) + .Annotation("MySql:CharSet", "utf8mb4") + }, + constraints: table => + { + table.PrimaryKey("PK_AtomizerJobErrors", x => x.Id); + table.ForeignKey( + name: "FK_AtomizerJobErrors_AtomizerJobs_JobId", + column: x => x.JobId, + principalTable: "AtomizerJobs", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }) + .Annotation("MySql:CharSet", "utf8mb4"); + + migrationBuilder.CreateIndex( + name: "IX_AtomizerJobErrors_JobId", + table: "AtomizerJobErrors", + column: "JobId"); + + migrationBuilder.CreateIndex( + name: "IX_AtomizerSchedules_JobKey", + table: "AtomizerSchedules", + column: "JobKey", + unique: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "AtomizerJobErrors"); + + migrationBuilder.DropTable( + name: "AtomizerSchedules"); + + migrationBuilder.DropTable( + name: "Products"); + + migrationBuilder.DropTable( + name: "AtomizerJobs"); + } + } +} diff --git a/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/ExampleMySqlContextModelSnapshot.cs b/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/ExampleMySqlContextModelSnapshot.cs index 1840d7b..7a03282 100644 --- a/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/ExampleMySqlContextModelSnapshot.cs +++ b/samples/Atomizer.EFCore.Example/Data/MySql/Migrations/ExampleMySqlContextModelSnapshot.cs @@ -72,6 +72,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasMaxLength(512) .HasColumnType("varchar(512)"); + b.Property("PartitionKey") + .HasMaxLength(255) + .HasColumnType("varchar(255)"); + b.Property("Payload") .IsRequired() .HasColumnType("longtext"); @@ -83,8 +87,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("QueueKey") .IsRequired() - .HasMaxLength(512) - .HasColumnType("varchar(512)"); + .HasMaxLength(100) + .HasColumnType("varchar(100)"); b.Property("RetryIntervals") .IsRequired() @@ -92,12 +96,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("varchar(4096)"); b.Property("ScheduleJobKey") - .HasMaxLength(512) - .HasColumnType("varchar(512)"); + .HasMaxLength(255) + .HasColumnType("varchar(255)"); b.Property("ScheduledAt") .HasColumnType("datetime(6)"); + b.Property("SequenceNumber") + .HasColumnType("bigint"); + b.Property("Status") .HasColumnType("int"); diff --git a/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20250827145628_Initial.Designer.cs b/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20250827145628_Initial.Designer.cs deleted file mode 100644 index f4bcd5b..0000000 --- a/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20250827145628_Initial.Designer.cs +++ /dev/null @@ -1,240 +0,0 @@ -// -using System; -using Atomizer.EFCore.Example.Data.Postgres; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; - -#nullable disable - -namespace Atomizer.EFCore.Example.Data.Postgres.Migrations -{ - [DbContext(typeof(ExamplePostgresContext))] - [Migration("20250827145628_Initial")] - partial class Initial - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "9.0.8") - .HasAnnotation("Relational:MaxIdentifierLength", 63); - - NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); - - modelBuilder.Entity("Atomizer.EFCore.Example.Entities.Product", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("CreatedAt") - .HasColumnType("timestamp with time zone"); - - b.Property("Name") - .IsRequired() - .HasColumnType("text"); - - b.Property("Price") - .HasColumnType("numeric"); - - b.Property("Quantity") - .HasColumnType("integer"); - - b.HasKey("Id"); - - b.ToTable("Products"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("Attempts") - .HasColumnType("integer"); - - b.Property("CompletedAt") - .HasColumnType("timestamp with time zone"); - - b.Property("CreatedAt") - .HasColumnType("timestamp with time zone"); - - b.Property("FailedAt") - .HasColumnType("timestamp with time zone"); - - b.Property("IdempotencyKey") - .HasMaxLength(512) - .HasColumnType("character varying(512)"); - - b.Property("LeaseToken") - .HasMaxLength(512) - .HasColumnType("character varying(512)"); - - b.Property("Payload") - .IsRequired() - .HasColumnType("text"); - - b.Property("PayloadType") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("character varying(1024)"); - - b.Property("QueueKey") - .IsRequired() - .HasMaxLength(512) - .HasColumnType("character varying(512)"); - - b.Property("RetryIntervals") - .IsRequired() - .HasMaxLength(4096) - .HasColumnType("character varying(4096)"); - - b.Property("ScheduleJobKey") - .HasMaxLength(512) - .HasColumnType("character varying(512)"); - - b.Property("ScheduledAt") - .HasColumnType("timestamp with time zone"); - - b.Property("Status") - .HasColumnType("integer"); - - b.Property("UpdatedAt") - .HasColumnType("timestamp with time zone"); - - b.Property("VisibleAt") - .HasColumnType("timestamp with time zone"); - - b.HasKey("Id"); - - b.ToTable("AtomizerJobs", "Atomizer"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobErrorEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("Attempt") - .HasColumnType("integer"); - - b.Property("CreatedAt") - .HasColumnType("timestamp with time zone"); - - b.Property("ErrorMessage") - .HasMaxLength(2048) - .HasColumnType("character varying(2048)"); - - b.Property("ExceptionType") - .HasMaxLength(1024) - .HasColumnType("character varying(1024)"); - - b.Property("JobId") - .HasColumnType("uuid"); - - b.Property("RuntimeIdentity") - .HasMaxLength(255) - .HasColumnType("character varying(255)"); - - b.Property("StackTrace") - .HasMaxLength(5120) - .HasColumnType("character varying(5120)"); - - b.HasKey("Id"); - - b.HasIndex("JobId"); - - b.ToTable("AtomizerJobErrors", "Atomizer"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerScheduleEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uuid"); - - b.Property("CreatedAt") - .HasColumnType("timestamp with time zone"); - - b.Property("Enabled") - .HasColumnType("boolean"); - - b.Property("JobKey") - .IsRequired() - .HasMaxLength(512) - .HasColumnType("character varying(512)"); - - b.Property("LastEnqueueAt") - .HasColumnType("timestamp with time zone"); - - b.Property("MaxCatchUp") - .HasColumnType("integer"); - - b.Property("MisfirePolicy") - .HasColumnType("integer"); - - b.Property("NextRunAt") - .HasColumnType("timestamp with time zone"); - - b.Property("Payload") - .IsRequired() - .HasColumnType("text"); - - b.Property("PayloadType") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("character varying(1024)"); - - b.Property("QueueKey") - .IsRequired() - .HasMaxLength(512) - .HasColumnType("character varying(512)"); - - b.Property("RetryIntervals") - .IsRequired() - .HasMaxLength(4096) - .HasColumnType("character varying(4096)"); - - b.Property("Schedule") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("character varying(1024)"); - - b.Property("TimeZone") - .IsRequired() - .HasMaxLength(64) - .HasColumnType("character varying(64)"); - - b.Property("UpdatedAt") - .HasColumnType("timestamp with time zone"); - - b.HasKey("Id"); - - b.ToTable("AtomizerSchedules", "Atomizer"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobErrorEntity", b => - { - b.HasOne("Atomizer.EntityFrameworkCore.Entities.AtomizerJobEntity", "Job") - .WithMany("Errors") - .HasForeignKey("JobId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.Navigation("Job"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobEntity", b => - { - b.Navigation("Errors"); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20260503193345_AddIndexForJobKeyOnSchedulesTable.cs b/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20260503193345_AddIndexForJobKeyOnSchedulesTable.cs deleted file mode 100644 index 85bbb0d..0000000 --- a/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20260503193345_AddIndexForJobKeyOnSchedulesTable.cs +++ /dev/null @@ -1,74 +0,0 @@ -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace Atomizer.EFCore.Example.Data.Postgres.Migrations -{ - /// - public partial class AddIndexForJobKeyOnSchedulesTable : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.AlterColumn( - name: "QueueKey", - schema: "Atomizer", - table: "AtomizerSchedules", - type: "character varying(100)", - maxLength: 100, - nullable: false, - oldClrType: typeof(string), - oldType: "character varying(512)", - oldMaxLength: 512); - - migrationBuilder.AlterColumn( - name: "JobKey", - schema: "Atomizer", - table: "AtomizerSchedules", - type: "character varying(255)", - maxLength: 255, - nullable: false, - oldClrType: typeof(string), - oldType: "character varying(512)", - oldMaxLength: 512); - - migrationBuilder.CreateIndex( - name: "IX_AtomizerSchedules_JobKey", - schema: "Atomizer", - table: "AtomizerSchedules", - column: "JobKey", - unique: true); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropIndex( - name: "IX_AtomizerSchedules_JobKey", - schema: "Atomizer", - table: "AtomizerSchedules"); - - migrationBuilder.AlterColumn( - name: "QueueKey", - schema: "Atomizer", - table: "AtomizerSchedules", - type: "character varying(512)", - maxLength: 512, - nullable: false, - oldClrType: typeof(string), - oldType: "character varying(100)", - oldMaxLength: 100); - - migrationBuilder.AlterColumn( - name: "JobKey", - schema: "Atomizer", - table: "AtomizerSchedules", - type: "character varying(512)", - maxLength: 512, - nullable: false, - oldClrType: typeof(string), - oldType: "character varying(255)", - oldMaxLength: 255); - } - } -} diff --git a/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20260503193345_AddIndexForJobKeyOnSchedulesTable.Designer.cs b/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20260504193605_Initial.Designer.cs similarity index 94% rename from samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20260503193345_AddIndexForJobKeyOnSchedulesTable.Designer.cs rename to samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20260504193605_Initial.Designer.cs index 667ace1..2b18890 100644 --- a/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20260503193345_AddIndexForJobKeyOnSchedulesTable.Designer.cs +++ b/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20260504193605_Initial.Designer.cs @@ -12,8 +12,8 @@ namespace Atomizer.EFCore.Example.Data.Postgres.Migrations { [DbContext(typeof(ExamplePostgresContext))] - [Migration("20260503193345_AddIndexForJobKeyOnSchedulesTable")] - partial class AddIndexForJobKeyOnSchedulesTable + [Migration("20260504193605_Initial")] + partial class Initial { /// protected override void BuildTargetModel(ModelBuilder modelBuilder) @@ -75,6 +75,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasMaxLength(512) .HasColumnType("character varying(512)"); + b.Property("PartitionKey") + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + b.Property("Payload") .IsRequired() .HasColumnType("text"); @@ -86,8 +90,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("QueueKey") .IsRequired() - .HasMaxLength(512) - .HasColumnType("character varying(512)"); + .HasMaxLength(100) + .HasColumnType("character varying(100)"); b.Property("RetryIntervals") .IsRequired() @@ -95,12 +99,15 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("character varying(4096)"); b.Property("ScheduleJobKey") - .HasMaxLength(512) - .HasColumnType("character varying(512)"); + .HasMaxLength(255) + .HasColumnType("character varying(255)"); b.Property("ScheduledAt") .HasColumnType("timestamp with time zone"); + b.Property("SequenceNumber") + .HasColumnType("bigint"); + b.Property("Status") .HasColumnType("integer"); diff --git a/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20250827145628_Initial.cs b/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20260504193605_Initial.cs similarity index 65% rename from samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20250827145628_Initial.cs rename to samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20260504193605_Initial.cs index 93ba47b..2503b82 100644 --- a/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20250827145628_Initial.cs +++ b/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/20260504193605_Initial.cs @@ -11,7 +11,8 @@ public partial class Initial : Migration /// protected override void Up(MigrationBuilder migrationBuilder) { - migrationBuilder.EnsureSchema(name: "Atomizer"); + migrationBuilder.EnsureSchema( + name: "Atomizer"); migrationBuilder.CreateTable( name: "AtomizerJobs", @@ -19,43 +20,28 @@ protected override void Up(MigrationBuilder migrationBuilder) columns: table => new { Id = table.Column(type: "uuid", nullable: false), - QueueKey = table.Column(type: "character varying(512)", maxLength: 512, nullable: false), - PayloadType = table.Column( - type: "character varying(1024)", - maxLength: 1024, - nullable: false - ), + QueueKey = table.Column(type: "character varying(100)", maxLength: 100, nullable: false), + PayloadType = table.Column(type: "character varying(1024)", maxLength: 1024, nullable: false), Payload = table.Column(type: "text", nullable: false), ScheduledAt = table.Column(type: "timestamp with time zone", nullable: false), VisibleAt = table.Column(type: "timestamp with time zone", nullable: true), Status = table.Column(type: "integer", nullable: false), Attempts = table.Column(type: "integer", nullable: false), - RetryIntervals = table.Column( - type: "character varying(4096)", - maxLength: 4096, - nullable: false - ), + RetryIntervals = table.Column(type: "character varying(4096)", maxLength: 4096, nullable: false), CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), UpdatedAt = table.Column(type: "timestamp with time zone", nullable: false), CompletedAt = table.Column(type: "timestamp with time zone", nullable: true), FailedAt = table.Column(type: "timestamp with time zone", nullable: true), LeaseToken = table.Column(type: "character varying(512)", maxLength: 512, nullable: true), - ScheduleJobKey = table.Column( - type: "character varying(512)", - maxLength: 512, - nullable: true - ), - IdempotencyKey = table.Column( - type: "character varying(512)", - maxLength: 512, - nullable: true - ), + ScheduleJobKey = table.Column(type: "character varying(255)", maxLength: 255, nullable: true), + IdempotencyKey = table.Column(type: "character varying(512)", maxLength: 512, nullable: true), + PartitionKey = table.Column(type: "character varying(255)", maxLength: 255, nullable: true), + SequenceNumber = table.Column(type: "bigint", nullable: true) }, constraints: table => { table.PrimaryKey("PK_AtomizerJobs", x => x.Id); - } - ); + }); migrationBuilder.CreateTable( name: "AtomizerSchedules", @@ -63,34 +49,25 @@ protected override void Up(MigrationBuilder migrationBuilder) columns: table => new { Id = table.Column(type: "uuid", nullable: false), - JobKey = table.Column(type: "character varying(512)", maxLength: 512, nullable: false), - QueueKey = table.Column(type: "character varying(512)", maxLength: 512, nullable: false), - PayloadType = table.Column( - type: "character varying(1024)", - maxLength: 1024, - nullable: false - ), + JobKey = table.Column(type: "character varying(255)", maxLength: 255, nullable: false), + QueueKey = table.Column(type: "character varying(100)", maxLength: 100, nullable: false), + PayloadType = table.Column(type: "character varying(1024)", maxLength: 1024, nullable: false), Payload = table.Column(type: "text", nullable: false), Schedule = table.Column(type: "character varying(1024)", maxLength: 1024, nullable: false), TimeZone = table.Column(type: "character varying(64)", maxLength: 64, nullable: false), MisfirePolicy = table.Column(type: "integer", nullable: false), MaxCatchUp = table.Column(type: "integer", nullable: false), Enabled = table.Column(type: "boolean", nullable: false), - RetryIntervals = table.Column( - type: "character varying(4096)", - maxLength: 4096, - nullable: false - ), + RetryIntervals = table.Column(type: "character varying(4096)", maxLength: 4096, nullable: false), NextRunAt = table.Column(type: "timestamp with time zone", nullable: false), LastEnqueueAt = table.Column(type: "timestamp with time zone", nullable: true), CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), - UpdatedAt = table.Column(type: "timestamp with time zone", nullable: false), + UpdatedAt = table.Column(type: "timestamp with time zone", nullable: false) }, constraints: table => { table.PrimaryKey("PK_AtomizerSchedules", x => x.Id); - } - ); + }); migrationBuilder.CreateTable( name: "Products", @@ -100,13 +77,12 @@ protected override void Up(MigrationBuilder migrationBuilder) Name = table.Column(type: "text", nullable: false), Price = table.Column(type: "numeric", nullable: false), CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), - Quantity = table.Column(type: "integer", nullable: false), + Quantity = table.Column(type: "integer", nullable: false) }, constraints: table => { table.PrimaryKey("PK_Products", x => x.Id); - } - ); + }); migrationBuilder.CreateTable( name: "AtomizerJobErrors", @@ -115,24 +91,12 @@ protected override void Up(MigrationBuilder migrationBuilder) { Id = table.Column(type: "uuid", nullable: false), JobId = table.Column(type: "uuid", nullable: false), - ErrorMessage = table.Column( - type: "character varying(2048)", - maxLength: 2048, - nullable: true - ), + ErrorMessage = table.Column(type: "character varying(2048)", maxLength: 2048, nullable: true), StackTrace = table.Column(type: "character varying(5120)", maxLength: 5120, nullable: true), - ExceptionType = table.Column( - type: "character varying(1024)", - maxLength: 1024, - nullable: true - ), + ExceptionType = table.Column(type: "character varying(1024)", maxLength: 1024, nullable: true), CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), Attempt = table.Column(type: "integer", nullable: false), - RuntimeIdentity = table.Column( - type: "character varying(255)", - maxLength: 255, - nullable: true - ), + RuntimeIdentity = table.Column(type: "character varying(255)", maxLength: 255, nullable: true) }, constraints: table => { @@ -143,29 +107,40 @@ protected override void Up(MigrationBuilder migrationBuilder) principalSchema: "Atomizer", principalTable: "AtomizerJobs", principalColumn: "Id", - onDelete: ReferentialAction.Cascade - ); - } - ); + onDelete: ReferentialAction.Cascade); + }); migrationBuilder.CreateIndex( name: "IX_AtomizerJobErrors_JobId", schema: "Atomizer", table: "AtomizerJobErrors", - column: "JobId" - ); + column: "JobId"); + + migrationBuilder.CreateIndex( + name: "IX_AtomizerSchedules_JobKey", + schema: "Atomizer", + table: "AtomizerSchedules", + column: "JobKey", + unique: true); } /// protected override void Down(MigrationBuilder migrationBuilder) { - migrationBuilder.DropTable(name: "AtomizerJobErrors", schema: "Atomizer"); + migrationBuilder.DropTable( + name: "AtomizerJobErrors", + schema: "Atomizer"); - migrationBuilder.DropTable(name: "AtomizerSchedules", schema: "Atomizer"); + migrationBuilder.DropTable( + name: "AtomizerSchedules", + schema: "Atomizer"); - migrationBuilder.DropTable(name: "Products"); + migrationBuilder.DropTable( + name: "Products"); - migrationBuilder.DropTable(name: "AtomizerJobs", schema: "Atomizer"); + migrationBuilder.DropTable( + name: "AtomizerJobs", + schema: "Atomizer"); } } } diff --git a/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/ExamplePostgresContextModelSnapshot.cs b/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/ExamplePostgresContextModelSnapshot.cs index 82b44a8..fc49076 100644 --- a/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/ExamplePostgresContextModelSnapshot.cs +++ b/samples/Atomizer.EFCore.Example/Data/Postgres/Migrations/ExamplePostgresContextModelSnapshot.cs @@ -72,6 +72,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasMaxLength(512) .HasColumnType("character varying(512)"); + b.Property("PartitionKey") + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + b.Property("Payload") .IsRequired() .HasColumnType("text"); @@ -83,8 +87,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("QueueKey") .IsRequired() - .HasMaxLength(512) - .HasColumnType("character varying(512)"); + .HasMaxLength(100) + .HasColumnType("character varying(100)"); b.Property("RetryIntervals") .IsRequired() @@ -92,12 +96,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("character varying(4096)"); b.Property("ScheduleJobKey") - .HasMaxLength(512) - .HasColumnType("character varying(512)"); + .HasMaxLength(255) + .HasColumnType("character varying(255)"); b.Property("ScheduledAt") .HasColumnType("timestamp with time zone"); + b.Property("SequenceNumber") + .HasColumnType("bigint"); + b.Property("Status") .HasColumnType("integer"); diff --git a/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20250827145633_Initial.Designer.cs b/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20250827145633_Initial.Designer.cs deleted file mode 100644 index 47ca2f9..0000000 --- a/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20250827145633_Initial.Designer.cs +++ /dev/null @@ -1,240 +0,0 @@ -// -using System; -using Atomizer.EFCore.Example.Data.SqlServer; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Metadata; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; - -#nullable disable - -namespace Atomizer.EFCore.Example.Data.SqlServer.Migrations -{ - [DbContext(typeof(ExampleSqlServerContext))] - [Migration("20250827145633_Initial")] - partial class Initial - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder - .HasAnnotation("ProductVersion", "9.0.8") - .HasAnnotation("Relational:MaxIdentifierLength", 128); - - SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); - - modelBuilder.Entity("Atomizer.EFCore.Example.Entities.Product", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uniqueidentifier"); - - b.Property("CreatedAt") - .HasColumnType("datetime2"); - - b.Property("Name") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("Price") - .HasColumnType("decimal(18,2)"); - - b.Property("Quantity") - .HasColumnType("int"); - - b.HasKey("Id"); - - b.ToTable("Products"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uniqueidentifier"); - - b.Property("Attempts") - .HasColumnType("int"); - - b.Property("CompletedAt") - .HasColumnType("datetimeoffset"); - - b.Property("CreatedAt") - .HasColumnType("datetimeoffset"); - - b.Property("FailedAt") - .HasColumnType("datetimeoffset"); - - b.Property("IdempotencyKey") - .HasMaxLength(512) - .HasColumnType("nvarchar(512)"); - - b.Property("LeaseToken") - .HasMaxLength(512) - .HasColumnType("nvarchar(512)"); - - b.Property("Payload") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("PayloadType") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("nvarchar(1024)"); - - b.Property("QueueKey") - .IsRequired() - .HasMaxLength(512) - .HasColumnType("nvarchar(512)"); - - b.Property("RetryIntervals") - .IsRequired() - .HasMaxLength(4096) - .HasColumnType("nvarchar(max)"); - - b.Property("ScheduleJobKey") - .HasMaxLength(512) - .HasColumnType("nvarchar(512)"); - - b.Property("ScheduledAt") - .HasColumnType("datetimeoffset"); - - b.Property("Status") - .HasColumnType("int"); - - b.Property("UpdatedAt") - .HasColumnType("datetimeoffset"); - - b.Property("VisibleAt") - .HasColumnType("datetimeoffset"); - - b.HasKey("Id"); - - b.ToTable("AtomizerJobs", "Atomizer"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobErrorEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uniqueidentifier"); - - b.Property("Attempt") - .HasColumnType("int"); - - b.Property("CreatedAt") - .HasColumnType("datetimeoffset"); - - b.Property("ErrorMessage") - .HasMaxLength(2048) - .HasColumnType("nvarchar(2048)"); - - b.Property("ExceptionType") - .HasMaxLength(1024) - .HasColumnType("nvarchar(1024)"); - - b.Property("JobId") - .HasColumnType("uniqueidentifier"); - - b.Property("RuntimeIdentity") - .HasMaxLength(255) - .HasColumnType("nvarchar(255)"); - - b.Property("StackTrace") - .HasMaxLength(5120) - .HasColumnType("nvarchar(max)"); - - b.HasKey("Id"); - - b.HasIndex("JobId"); - - b.ToTable("AtomizerJobErrors", "Atomizer"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerScheduleEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("uniqueidentifier"); - - b.Property("CreatedAt") - .HasColumnType("datetimeoffset"); - - b.Property("Enabled") - .HasColumnType("bit"); - - b.Property("JobKey") - .IsRequired() - .HasMaxLength(512) - .HasColumnType("nvarchar(512)"); - - b.Property("LastEnqueueAt") - .HasColumnType("datetimeoffset"); - - b.Property("MaxCatchUp") - .HasColumnType("int"); - - b.Property("MisfirePolicy") - .HasColumnType("int"); - - b.Property("NextRunAt") - .HasColumnType("datetimeoffset"); - - b.Property("Payload") - .IsRequired() - .HasColumnType("nvarchar(max)"); - - b.Property("PayloadType") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("nvarchar(1024)"); - - b.Property("QueueKey") - .IsRequired() - .HasMaxLength(512) - .HasColumnType("nvarchar(512)"); - - b.Property("RetryIntervals") - .IsRequired() - .HasMaxLength(4096) - .HasColumnType("nvarchar(max)"); - - b.Property("Schedule") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("nvarchar(1024)"); - - b.Property("TimeZone") - .IsRequired() - .HasMaxLength(64) - .HasColumnType("nvarchar(64)"); - - b.Property("UpdatedAt") - .HasColumnType("datetimeoffset"); - - b.HasKey("Id"); - - b.ToTable("AtomizerSchedules", "Atomizer"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobErrorEntity", b => - { - b.HasOne("Atomizer.EntityFrameworkCore.Entities.AtomizerJobEntity", "Job") - .WithMany("Errors") - .HasForeignKey("JobId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.Navigation("Job"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobEntity", b => - { - b.Navigation("Errors"); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20260503193351_AddIndexForJobKeyOnSchedulesTable.cs b/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20260503193351_AddIndexForJobKeyOnSchedulesTable.cs deleted file mode 100644 index d0f0558..0000000 --- a/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20260503193351_AddIndexForJobKeyOnSchedulesTable.cs +++ /dev/null @@ -1,74 +0,0 @@ -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace Atomizer.EFCore.Example.Data.SqlServer.Migrations -{ - /// - public partial class AddIndexForJobKeyOnSchedulesTable : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.AlterColumn( - name: "QueueKey", - schema: "Atomizer", - table: "AtomizerSchedules", - type: "nvarchar(100)", - maxLength: 100, - nullable: false, - oldClrType: typeof(string), - oldType: "nvarchar(512)", - oldMaxLength: 512); - - migrationBuilder.AlterColumn( - name: "JobKey", - schema: "Atomizer", - table: "AtomizerSchedules", - type: "nvarchar(255)", - maxLength: 255, - nullable: false, - oldClrType: typeof(string), - oldType: "nvarchar(512)", - oldMaxLength: 512); - - migrationBuilder.CreateIndex( - name: "IX_AtomizerSchedules_JobKey", - schema: "Atomizer", - table: "AtomizerSchedules", - column: "JobKey", - unique: true); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropIndex( - name: "IX_AtomizerSchedules_JobKey", - schema: "Atomizer", - table: "AtomizerSchedules"); - - migrationBuilder.AlterColumn( - name: "QueueKey", - schema: "Atomizer", - table: "AtomizerSchedules", - type: "nvarchar(512)", - maxLength: 512, - nullable: false, - oldClrType: typeof(string), - oldType: "nvarchar(100)", - oldMaxLength: 100); - - migrationBuilder.AlterColumn( - name: "JobKey", - schema: "Atomizer", - table: "AtomizerSchedules", - type: "nvarchar(512)", - maxLength: 512, - nullable: false, - oldClrType: typeof(string), - oldType: "nvarchar(255)", - oldMaxLength: 255); - } - } -} diff --git a/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20260503193351_AddIndexForJobKeyOnSchedulesTable.Designer.cs b/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20260504193610_Initial.Designer.cs similarity index 94% rename from samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20260503193351_AddIndexForJobKeyOnSchedulesTable.Designer.cs rename to samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20260504193610_Initial.Designer.cs index 92cc882..44caae1 100644 --- a/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20260503193351_AddIndexForJobKeyOnSchedulesTable.Designer.cs +++ b/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20260504193610_Initial.Designer.cs @@ -12,8 +12,8 @@ namespace Atomizer.EFCore.Example.Data.SqlServer.Migrations { [DbContext(typeof(ExampleSqlServerContext))] - [Migration("20260503193351_AddIndexForJobKeyOnSchedulesTable")] - partial class AddIndexForJobKeyOnSchedulesTable + [Migration("20260504193610_Initial")] + partial class Initial { /// protected override void BuildTargetModel(ModelBuilder modelBuilder) @@ -75,6 +75,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasMaxLength(512) .HasColumnType("nvarchar(512)"); + b.Property("PartitionKey") + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + b.Property("Payload") .IsRequired() .HasColumnType("nvarchar(max)"); @@ -86,8 +90,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("QueueKey") .IsRequired() - .HasMaxLength(512) - .HasColumnType("nvarchar(512)"); + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); b.Property("RetryIntervals") .IsRequired() @@ -95,12 +99,15 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("nvarchar(max)"); b.Property("ScheduleJobKey") - .HasMaxLength(512) - .HasColumnType("nvarchar(512)"); + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); b.Property("ScheduledAt") .HasColumnType("datetimeoffset"); + b.Property("SequenceNumber") + .HasColumnType("bigint"); + b.Property("Status") .HasColumnType("int"); diff --git a/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20250827145633_Initial.cs b/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20260504193610_Initial.cs similarity index 81% rename from samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20250827145633_Initial.cs rename to samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20260504193610_Initial.cs index 35aa82b..2c1b131 100644 --- a/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20250827145633_Initial.cs +++ b/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/20260504193610_Initial.cs @@ -11,7 +11,8 @@ public partial class Initial : Migration /// protected override void Up(MigrationBuilder migrationBuilder) { - migrationBuilder.EnsureSchema(name: "Atomizer"); + migrationBuilder.EnsureSchema( + name: "Atomizer"); migrationBuilder.CreateTable( name: "AtomizerJobs", @@ -19,7 +20,7 @@ protected override void Up(MigrationBuilder migrationBuilder) columns: table => new { Id = table.Column(type: "uniqueidentifier", nullable: false), - QueueKey = table.Column(type: "nvarchar(512)", maxLength: 512, nullable: false), + QueueKey = table.Column(type: "nvarchar(100)", maxLength: 100, nullable: false), PayloadType = table.Column(type: "nvarchar(1024)", maxLength: 1024, nullable: false), Payload = table.Column(type: "nvarchar(max)", nullable: false), ScheduledAt = table.Column(type: "datetimeoffset", nullable: false), @@ -32,14 +33,15 @@ protected override void Up(MigrationBuilder migrationBuilder) CompletedAt = table.Column(type: "datetimeoffset", nullable: true), FailedAt = table.Column(type: "datetimeoffset", nullable: true), LeaseToken = table.Column(type: "nvarchar(512)", maxLength: 512, nullable: true), - ScheduleJobKey = table.Column(type: "nvarchar(512)", maxLength: 512, nullable: true), + ScheduleJobKey = table.Column(type: "nvarchar(255)", maxLength: 255, nullable: true), IdempotencyKey = table.Column(type: "nvarchar(512)", maxLength: 512, nullable: true), + PartitionKey = table.Column(type: "nvarchar(255)", maxLength: 255, nullable: true), + SequenceNumber = table.Column(type: "bigint", nullable: true) }, constraints: table => { table.PrimaryKey("PK_AtomizerJobs", x => x.Id); - } - ); + }); migrationBuilder.CreateTable( name: "AtomizerSchedules", @@ -47,8 +49,8 @@ protected override void Up(MigrationBuilder migrationBuilder) columns: table => new { Id = table.Column(type: "uniqueidentifier", nullable: false), - JobKey = table.Column(type: "nvarchar(512)", maxLength: 512, nullable: false), - QueueKey = table.Column(type: "nvarchar(512)", maxLength: 512, nullable: false), + JobKey = table.Column(type: "nvarchar(255)", maxLength: 255, nullable: false), + QueueKey = table.Column(type: "nvarchar(100)", maxLength: 100, nullable: false), PayloadType = table.Column(type: "nvarchar(1024)", maxLength: 1024, nullable: false), Payload = table.Column(type: "nvarchar(max)", nullable: false), Schedule = table.Column(type: "nvarchar(1024)", maxLength: 1024, nullable: false), @@ -60,13 +62,12 @@ protected override void Up(MigrationBuilder migrationBuilder) NextRunAt = table.Column(type: "datetimeoffset", nullable: false), LastEnqueueAt = table.Column(type: "datetimeoffset", nullable: true), CreatedAt = table.Column(type: "datetimeoffset", nullable: false), - UpdatedAt = table.Column(type: "datetimeoffset", nullable: false), + UpdatedAt = table.Column(type: "datetimeoffset", nullable: false) }, constraints: table => { table.PrimaryKey("PK_AtomizerSchedules", x => x.Id); - } - ); + }); migrationBuilder.CreateTable( name: "Products", @@ -76,13 +77,12 @@ protected override void Up(MigrationBuilder migrationBuilder) Name = table.Column(type: "nvarchar(max)", nullable: false), Price = table.Column(type: "decimal(18,2)", nullable: false), CreatedAt = table.Column(type: "datetime2", nullable: false), - Quantity = table.Column(type: "int", nullable: false), + Quantity = table.Column(type: "int", nullable: false) }, constraints: table => { table.PrimaryKey("PK_Products", x => x.Id); - } - ); + }); migrationBuilder.CreateTable( name: "AtomizerJobErrors", @@ -96,7 +96,7 @@ protected override void Up(MigrationBuilder migrationBuilder) ExceptionType = table.Column(type: "nvarchar(1024)", maxLength: 1024, nullable: true), CreatedAt = table.Column(type: "datetimeoffset", nullable: false), Attempt = table.Column(type: "int", nullable: false), - RuntimeIdentity = table.Column(type: "nvarchar(255)", maxLength: 255, nullable: true), + RuntimeIdentity = table.Column(type: "nvarchar(255)", maxLength: 255, nullable: true) }, constraints: table => { @@ -107,29 +107,40 @@ protected override void Up(MigrationBuilder migrationBuilder) principalSchema: "Atomizer", principalTable: "AtomizerJobs", principalColumn: "Id", - onDelete: ReferentialAction.Cascade - ); - } - ); + onDelete: ReferentialAction.Cascade); + }); migrationBuilder.CreateIndex( name: "IX_AtomizerJobErrors_JobId", schema: "Atomizer", table: "AtomizerJobErrors", - column: "JobId" - ); + column: "JobId"); + + migrationBuilder.CreateIndex( + name: "IX_AtomizerSchedules_JobKey", + schema: "Atomizer", + table: "AtomizerSchedules", + column: "JobKey", + unique: true); } /// protected override void Down(MigrationBuilder migrationBuilder) { - migrationBuilder.DropTable(name: "AtomizerJobErrors", schema: "Atomizer"); + migrationBuilder.DropTable( + name: "AtomizerJobErrors", + schema: "Atomizer"); - migrationBuilder.DropTable(name: "AtomizerSchedules", schema: "Atomizer"); + migrationBuilder.DropTable( + name: "AtomizerSchedules", + schema: "Atomizer"); - migrationBuilder.DropTable(name: "Products"); + migrationBuilder.DropTable( + name: "Products"); - migrationBuilder.DropTable(name: "AtomizerJobs", schema: "Atomizer"); + migrationBuilder.DropTable( + name: "AtomizerJobs", + schema: "Atomizer"); } } } diff --git a/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/ExampleSqlServerContextModelSnapshot.cs b/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/ExampleSqlServerContextModelSnapshot.cs index 0d4e211..48ffccb 100644 --- a/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/ExampleSqlServerContextModelSnapshot.cs +++ b/samples/Atomizer.EFCore.Example/Data/SqlServer/Migrations/ExampleSqlServerContextModelSnapshot.cs @@ -72,6 +72,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasMaxLength(512) .HasColumnType("nvarchar(512)"); + b.Property("PartitionKey") + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + b.Property("Payload") .IsRequired() .HasColumnType("nvarchar(max)"); @@ -83,8 +87,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("QueueKey") .IsRequired() - .HasMaxLength(512) - .HasColumnType("nvarchar(512)"); + .HasMaxLength(100) + .HasColumnType("nvarchar(100)"); b.Property("RetryIntervals") .IsRequired() @@ -92,12 +96,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("nvarchar(max)"); b.Property("ScheduleJobKey") - .HasMaxLength(512) - .HasColumnType("nvarchar(512)"); + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); b.Property("ScheduledAt") .HasColumnType("datetimeoffset"); + b.Property("SequenceNumber") + .HasColumnType("bigint"); + b.Property("Status") .HasColumnType("int"); diff --git a/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20250827145638_Initial.Designer.cs b/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20250827145638_Initial.Designer.cs deleted file mode 100644 index d70f5c9..0000000 --- a/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20250827145638_Initial.Designer.cs +++ /dev/null @@ -1,235 +0,0 @@ -// -using System; -using Atomizer.EFCore.Example.Data.Sqlite; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; - -#nullable disable - -namespace Atomizer.EFCore.Example.Data.Sqlite.Migrations -{ - [DbContext(typeof(ExampleSqliteContext))] - [Migration("20250827145638_Initial")] - partial class Initial - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "9.0.8"); - - modelBuilder.Entity("Atomizer.EFCore.Example.Entities.Product", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("TEXT"); - - b.Property("CreatedAt") - .HasColumnType("TEXT"); - - b.Property("Name") - .IsRequired() - .HasColumnType("TEXT"); - - b.Property("Price") - .HasColumnType("TEXT"); - - b.Property("Quantity") - .HasColumnType("INTEGER"); - - b.HasKey("Id"); - - b.ToTable("Products"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("TEXT"); - - b.Property("Attempts") - .HasColumnType("INTEGER"); - - b.Property("CompletedAt") - .HasColumnType("TEXT"); - - b.Property("CreatedAt") - .HasColumnType("TEXT"); - - b.Property("FailedAt") - .HasColumnType("TEXT"); - - b.Property("IdempotencyKey") - .HasMaxLength(512) - .HasColumnType("TEXT"); - - b.Property("LeaseToken") - .HasMaxLength(512) - .HasColumnType("TEXT"); - - b.Property("Payload") - .IsRequired() - .HasColumnType("TEXT"); - - b.Property("PayloadType") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("TEXT"); - - b.Property("QueueKey") - .IsRequired() - .HasMaxLength(512) - .HasColumnType("TEXT"); - - b.Property("RetryIntervals") - .IsRequired() - .HasMaxLength(4096) - .HasColumnType("TEXT"); - - b.Property("ScheduleJobKey") - .HasMaxLength(512) - .HasColumnType("TEXT"); - - b.Property("ScheduledAt") - .HasColumnType("TEXT"); - - b.Property("Status") - .HasColumnType("INTEGER"); - - b.Property("UpdatedAt") - .HasColumnType("TEXT"); - - b.Property("VisibleAt") - .HasColumnType("TEXT"); - - b.HasKey("Id"); - - b.ToTable("AtomizerJobs", "Atomizer"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobErrorEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("TEXT"); - - b.Property("Attempt") - .HasColumnType("INTEGER"); - - b.Property("CreatedAt") - .HasColumnType("TEXT"); - - b.Property("ErrorMessage") - .HasMaxLength(2048) - .HasColumnType("TEXT"); - - b.Property("ExceptionType") - .HasMaxLength(1024) - .HasColumnType("TEXT"); - - b.Property("JobId") - .HasColumnType("TEXT"); - - b.Property("RuntimeIdentity") - .HasMaxLength(255) - .HasColumnType("TEXT"); - - b.Property("StackTrace") - .HasMaxLength(5120) - .HasColumnType("TEXT"); - - b.HasKey("Id"); - - b.HasIndex("JobId"); - - b.ToTable("AtomizerJobErrors", "Atomizer"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerScheduleEntity", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("TEXT"); - - b.Property("CreatedAt") - .HasColumnType("TEXT"); - - b.Property("Enabled") - .HasColumnType("INTEGER"); - - b.Property("JobKey") - .IsRequired() - .HasMaxLength(512) - .HasColumnType("TEXT"); - - b.Property("LastEnqueueAt") - .HasColumnType("TEXT"); - - b.Property("MaxCatchUp") - .HasColumnType("INTEGER"); - - b.Property("MisfirePolicy") - .HasColumnType("INTEGER"); - - b.Property("NextRunAt") - .HasColumnType("TEXT"); - - b.Property("Payload") - .IsRequired() - .HasColumnType("TEXT"); - - b.Property("PayloadType") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("TEXT"); - - b.Property("QueueKey") - .IsRequired() - .HasMaxLength(512) - .HasColumnType("TEXT"); - - b.Property("RetryIntervals") - .IsRequired() - .HasMaxLength(4096) - .HasColumnType("TEXT"); - - b.Property("Schedule") - .IsRequired() - .HasMaxLength(1024) - .HasColumnType("TEXT"); - - b.Property("TimeZone") - .IsRequired() - .HasMaxLength(64) - .HasColumnType("TEXT"); - - b.Property("UpdatedAt") - .HasColumnType("TEXT"); - - b.HasKey("Id"); - - b.ToTable("AtomizerSchedules", "Atomizer"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobErrorEntity", b => - { - b.HasOne("Atomizer.EntityFrameworkCore.Entities.AtomizerJobEntity", "Job") - .WithMany("Errors") - .HasForeignKey("JobId") - .OnDelete(DeleteBehavior.Cascade) - .IsRequired(); - - b.Navigation("Job"); - }); - - modelBuilder.Entity("Atomizer.EntityFrameworkCore.Entities.AtomizerJobEntity", b => - { - b.Navigation("Errors"); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20260503193354_AddIndexForJobKeyOnSchedulesTable.cs b/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20260503193354_AddIndexForJobKeyOnSchedulesTable.cs deleted file mode 100644 index 9ed54af..0000000 --- a/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20260503193354_AddIndexForJobKeyOnSchedulesTable.cs +++ /dev/null @@ -1,30 +0,0 @@ -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace Atomizer.EFCore.Example.Data.Sqlite.Migrations -{ - /// - public partial class AddIndexForJobKeyOnSchedulesTable : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.CreateIndex( - name: "IX_AtomizerSchedules_JobKey", - schema: "Atomizer", - table: "AtomizerSchedules", - column: "JobKey", - unique: true); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropIndex( - name: "IX_AtomizerSchedules_JobKey", - schema: "Atomizer", - table: "AtomizerSchedules"); - } - } -} diff --git a/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20260503193354_AddIndexForJobKeyOnSchedulesTable.Designer.cs b/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20260504193615_Initial.Designer.cs similarity index 95% rename from samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20260503193354_AddIndexForJobKeyOnSchedulesTable.Designer.cs rename to samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20260504193615_Initial.Designer.cs index 2fcf0a0..75eb913 100644 --- a/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20260503193354_AddIndexForJobKeyOnSchedulesTable.Designer.cs +++ b/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20260504193615_Initial.Designer.cs @@ -11,8 +11,8 @@ namespace Atomizer.EFCore.Example.Data.Sqlite.Migrations { [DbContext(typeof(ExampleSqliteContext))] - [Migration("20260503193354_AddIndexForJobKeyOnSchedulesTable")] - partial class AddIndexForJobKeyOnSchedulesTable + [Migration("20260504193615_Initial")] + partial class Initial { /// protected override void BuildTargetModel(ModelBuilder modelBuilder) @@ -70,6 +70,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasMaxLength(512) .HasColumnType("TEXT"); + b.Property("PartitionKey") + .HasMaxLength(255) + .HasColumnType("TEXT"); + b.Property("Payload") .IsRequired() .HasColumnType("TEXT"); @@ -81,7 +85,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) b.Property("QueueKey") .IsRequired() - .HasMaxLength(512) + .HasMaxLength(100) .HasColumnType("TEXT"); b.Property("RetryIntervals") @@ -90,12 +94,15 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("TEXT"); b.Property("ScheduleJobKey") - .HasMaxLength(512) + .HasMaxLength(255) .HasColumnType("TEXT"); b.Property("ScheduledAt") .HasColumnType("TEXT"); + b.Property("SequenceNumber") + .HasColumnType("INTEGER"); + b.Property("Status") .HasColumnType("INTEGER"); diff --git a/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20250827145638_Initial.cs b/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20260504193615_Initial.cs similarity index 81% rename from samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20250827145638_Initial.cs rename to samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20260504193615_Initial.cs index d79880b..6a85904 100644 --- a/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20250827145638_Initial.cs +++ b/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/20260504193615_Initial.cs @@ -11,7 +11,8 @@ public partial class Initial : Migration /// protected override void Up(MigrationBuilder migrationBuilder) { - migrationBuilder.EnsureSchema(name: "Atomizer"); + migrationBuilder.EnsureSchema( + name: "Atomizer"); migrationBuilder.CreateTable( name: "AtomizerJobs", @@ -19,7 +20,7 @@ protected override void Up(MigrationBuilder migrationBuilder) columns: table => new { Id = table.Column(type: "TEXT", nullable: false), - QueueKey = table.Column(type: "TEXT", maxLength: 512, nullable: false), + QueueKey = table.Column(type: "TEXT", maxLength: 100, nullable: false), PayloadType = table.Column(type: "TEXT", maxLength: 1024, nullable: false), Payload = table.Column(type: "TEXT", nullable: false), ScheduledAt = table.Column(type: "TEXT", nullable: false), @@ -32,14 +33,15 @@ protected override void Up(MigrationBuilder migrationBuilder) CompletedAt = table.Column(type: "TEXT", nullable: true), FailedAt = table.Column(type: "TEXT", nullable: true), LeaseToken = table.Column(type: "TEXT", maxLength: 512, nullable: true), - ScheduleJobKey = table.Column(type: "TEXT", maxLength: 512, nullable: true), + ScheduleJobKey = table.Column(type: "TEXT", maxLength: 255, nullable: true), IdempotencyKey = table.Column(type: "TEXT", maxLength: 512, nullable: true), + PartitionKey = table.Column(type: "TEXT", maxLength: 255, nullable: true), + SequenceNumber = table.Column(type: "INTEGER", nullable: true) }, constraints: table => { table.PrimaryKey("PK_AtomizerJobs", x => x.Id); - } - ); + }); migrationBuilder.CreateTable( name: "AtomizerSchedules", @@ -47,8 +49,8 @@ protected override void Up(MigrationBuilder migrationBuilder) columns: table => new { Id = table.Column(type: "TEXT", nullable: false), - JobKey = table.Column(type: "TEXT", maxLength: 512, nullable: false), - QueueKey = table.Column(type: "TEXT", maxLength: 512, nullable: false), + JobKey = table.Column(type: "TEXT", maxLength: 255, nullable: false), + QueueKey = table.Column(type: "TEXT", maxLength: 100, nullable: false), PayloadType = table.Column(type: "TEXT", maxLength: 1024, nullable: false), Payload = table.Column(type: "TEXT", nullable: false), Schedule = table.Column(type: "TEXT", maxLength: 1024, nullable: false), @@ -60,13 +62,12 @@ protected override void Up(MigrationBuilder migrationBuilder) NextRunAt = table.Column(type: "TEXT", nullable: false), LastEnqueueAt = table.Column(type: "TEXT", nullable: true), CreatedAt = table.Column(type: "TEXT", nullable: false), - UpdatedAt = table.Column(type: "TEXT", nullable: false), + UpdatedAt = table.Column(type: "TEXT", nullable: false) }, constraints: table => { table.PrimaryKey("PK_AtomizerSchedules", x => x.Id); - } - ); + }); migrationBuilder.CreateTable( name: "Products", @@ -76,13 +77,12 @@ protected override void Up(MigrationBuilder migrationBuilder) Name = table.Column(type: "TEXT", nullable: false), Price = table.Column(type: "TEXT", nullable: false), CreatedAt = table.Column(type: "TEXT", nullable: false), - Quantity = table.Column(type: "INTEGER", nullable: false), + Quantity = table.Column(type: "INTEGER", nullable: false) }, constraints: table => { table.PrimaryKey("PK_Products", x => x.Id); - } - ); + }); migrationBuilder.CreateTable( name: "AtomizerJobErrors", @@ -96,7 +96,7 @@ protected override void Up(MigrationBuilder migrationBuilder) ExceptionType = table.Column(type: "TEXT", maxLength: 1024, nullable: true), CreatedAt = table.Column(type: "TEXT", nullable: false), Attempt = table.Column(type: "INTEGER", nullable: false), - RuntimeIdentity = table.Column(type: "TEXT", maxLength: 255, nullable: true), + RuntimeIdentity = table.Column(type: "TEXT", maxLength: 255, nullable: true) }, constraints: table => { @@ -107,29 +107,40 @@ protected override void Up(MigrationBuilder migrationBuilder) principalSchema: "Atomizer", principalTable: "AtomizerJobs", principalColumn: "Id", - onDelete: ReferentialAction.Cascade - ); - } - ); + onDelete: ReferentialAction.Cascade); + }); migrationBuilder.CreateIndex( name: "IX_AtomizerJobErrors_JobId", schema: "Atomizer", table: "AtomizerJobErrors", - column: "JobId" - ); + column: "JobId"); + + migrationBuilder.CreateIndex( + name: "IX_AtomizerSchedules_JobKey", + schema: "Atomizer", + table: "AtomizerSchedules", + column: "JobKey", + unique: true); } /// protected override void Down(MigrationBuilder migrationBuilder) { - migrationBuilder.DropTable(name: "AtomizerJobErrors", schema: "Atomizer"); + migrationBuilder.DropTable( + name: "AtomizerJobErrors", + schema: "Atomizer"); - migrationBuilder.DropTable(name: "AtomizerSchedules", schema: "Atomizer"); + migrationBuilder.DropTable( + name: "AtomizerSchedules", + schema: "Atomizer"); - migrationBuilder.DropTable(name: "Products"); + migrationBuilder.DropTable( + name: "Products"); - migrationBuilder.DropTable(name: "AtomizerJobs", schema: "Atomizer"); + migrationBuilder.DropTable( + name: "AtomizerJobs", + schema: "Atomizer"); } } } diff --git a/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/ExampleSqliteContextModelSnapshot.cs b/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/ExampleSqliteContextModelSnapshot.cs index 958e552..a11ed5c 100644 --- a/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/ExampleSqliteContextModelSnapshot.cs +++ b/samples/Atomizer.EFCore.Example/Data/Sqlite/Migrations/ExampleSqliteContextModelSnapshot.cs @@ -67,6 +67,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasMaxLength(512) .HasColumnType("TEXT"); + b.Property("PartitionKey") + .HasMaxLength(255) + .HasColumnType("TEXT"); + b.Property("Payload") .IsRequired() .HasColumnType("TEXT"); @@ -78,7 +82,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.Property("QueueKey") .IsRequired() - .HasMaxLength(512) + .HasMaxLength(100) .HasColumnType("TEXT"); b.Property("RetryIntervals") @@ -87,12 +91,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("TEXT"); b.Property("ScheduleJobKey") - .HasMaxLength(512) + .HasMaxLength(255) .HasColumnType("TEXT"); b.Property("ScheduledAt") .HasColumnType("TEXT"); + b.Property("SequenceNumber") + .HasColumnType("INTEGER"); + b.Property("Status") .HasColumnType("INTEGER"); From 13e0186dc4c5ecf183b9cd354233c41def4925d8 Mon Sep 17 00:00:00 2001 From: mnbuhl Date: Mon, 4 May 2026 21:40:34 +0200 Subject: [PATCH 53/53] feat(samples): add FIFO partitioned job example and update README - Add ProcessStockEventJob handler demonstrating PartitionKey usage: stock events for the same product are processed in strict FIFO order - Add POST /stock-events endpoint that enqueues with PartitionKey set to the ProductId, serializing per-product processing - README: move FIFO from Planned Features to Features, add Quick Start section 5 explaining partition key usage with a code example Co-Authored-By: Claude Sonnet 4.6 --- README.md | 17 ++++++++-- .../Handlers/ProcessStockEventJob.cs | 34 +++++++++++++++++++ samples/Atomizer.EFCore.Example/Program.cs | 14 ++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 samples/Atomizer.EFCore.Example/Handlers/ProcessStockEventJob.cs diff --git a/README.md b/README.md index b2881d1..611ee24 100644 --- a/README.md +++ b/README.md @@ -21,12 +21,12 @@ Atomizer is a modern, high-performance job scheduling and queueing framework for - 🛑 **Graceful Shutdown** — Ensure in-flight jobs finish and pending batched jobs are safely released for re-processing during shutdowns. - 📦 **Batch Processing** — Tune throughput with batch size and parallelism settings per queue. - ⏳ **Visibility Timeout** — Prevent job duplication by locking jobs during processing. +- 🕒 **FIFO Partitioned Processing** — Guarantee strict in-order, one-at-a-time execution per partition key (e.g. per customer, per entity). - 🧪 **In-Memory Driver** — Perfect for local development and testing; spin up queues instantly with zero setup. - 🔔 **ASP.NET Core Integration** — Works with DI, logging, and modern C# idioms. ## Planned Features - 📈 **Dashboard** — Live monitoring, retry/dead-letter management, and operational insights. -- 🕒 **FIFO Processing** — Guarantee jobs are processed in strict order, without overlap. - ⚡ **Redis Driver** — Lightning-fast, distributed, in-memory queues for massive scale. ## Quick Start @@ -130,7 +130,20 @@ app.MapPost( ); ``` -### 5. Schedule Recurring Jobs +### 5. FIFO Processing (Partitioned Jobs) +To guarantee jobs for the same entity execute one-at-a-time in enqueue order, assign a `PartitionKey`: + +```csharp +// All stock events for the same product are processed in strict FIFO order. +await atomizerClient.EnqueueAsync( + new StockEvent(productId, "restock", delta: 50), + options => options.PartitionKey = new PartitionKey(productId.ToString()) +); +``` + +Jobs sharing the same `PartitionKey` and queue are serialized: the next job in the partition only starts after the previous one completes (or fails and is rescheduled). Unpartitioned jobs in the same queue are unaffected and continue to process in parallel. + +### 6. Schedule Recurring Jobs in Program.cs: ```csharp ... diff --git a/samples/Atomizer.EFCore.Example/Handlers/ProcessStockEventJob.cs b/samples/Atomizer.EFCore.Example/Handlers/ProcessStockEventJob.cs new file mode 100644 index 0000000..3726ce7 --- /dev/null +++ b/samples/Atomizer.EFCore.Example/Handlers/ProcessStockEventJob.cs @@ -0,0 +1,34 @@ +using Atomizer.EFCore.Example.Data.Postgres; + +namespace Atomizer.EFCore.Example.Handlers; + +public record StockEvent(Guid ProductId, string EventType, int Delta); + +public class ProcessStockEventJob(ExamplePostgresContext dbContext, ILogger logger) + : IAtomizerJob +{ + public async Task HandleAsync(StockEvent payload, JobContext context) + { + var product = await dbContext.Products.FindAsync( + [payload.ProductId], + context.CancellationToken + ); + + if (product == null) + { + logger.LogWarning("Product {ProductId} not found, skipping stock event", payload.ProductId); + return; + } + + product.Quantity += payload.Delta; + await dbContext.SaveChangesAsync(context.CancellationToken); + + logger.LogInformation( + "Stock event '{EventType}' applied to product {ProductId}: delta={Delta}, new quantity={Quantity}", + payload.EventType, + payload.ProductId, + payload.Delta, + product.Quantity + ); + } +} diff --git a/samples/Atomizer.EFCore.Example/Program.cs b/samples/Atomizer.EFCore.Example/Program.cs index a619e8f..a7ff760 100644 --- a/samples/Atomizer.EFCore.Example/Program.cs +++ b/samples/Atomizer.EFCore.Example/Program.cs @@ -155,4 +155,18 @@ await atomizer.ScheduleRecurringAsync( } ); +// FIFO example: stock events for the same product are partitioned by ProductId, +// guaranteeing they execute one-at-a-time in enqueue order. +app.MapPost( + "/stock-events", + async ([FromServices] IAtomizerClient atomizerClient, [FromBody] StockEvent stockEvent) => + { + var jobId = await atomizerClient.EnqueueAsync( + stockEvent, + options => options.PartitionKey = new PartitionKey(stockEvent.ProductId.ToString()) + ); + return Results.Accepted($"/jobs/{jobId}"); + } +); + app.Run();