From 757d4642facd36ba72c7f7ff27e9f5f685d849f6 Mon Sep 17 00:00:00 2001 From: Marcus Kimpenhaus Date: Mon, 4 May 2026 00:33:17 +0200 Subject: [PATCH] feat(crds): introduce CrdInstallerSettings with builder and retry mechanism - Added `CrdInstallerSettingsBuilder` for configuring `CrdInstallerSettings` using a fluent API. - Introduced retry mechanism with exponential backoff for CRD installation to handle transient errors. - Updated templates and documentation to reflect new `WithOverwriteExisting` and `WithDeleteOnShutdown` APIs. - Added extensive unit tests for new settings, builder, and CRD installer behavior to ensure stability. --- docs/docs/operator/build-customization.mdx | 10 +- docs/docs/operator/utilities.mdx | 13 +- examples/Operator/Program.cs | 8 +- .../Builder/IOperatorBuilder.cs | 4 +- .../Crds/CrdInstallerSettings.cs | 8 +- .../Crds/CrdInstallerSettingsBuilder.cs | 36 ++++ .../CrdInstallerSettingsBuilderExtensions.cs | 36 ++++ .../Builder/OperatorBuilder.cs | 9 +- src/KubeOps.Operator/Crds/CrdInstaller.cs | 201 ++++++++++++++---- .../LeaderElectionBackgroundService.cs | 7 +- .../Retry/ExponentialRetryBackoff.cs | 12 ++ .../Watcher/ResourceWatcher{TEntity}.cs | 5 +- .../Templates/EmptyOperator.CSharp/Program.cs | 6 +- .../EmptyWebOperator.CSharp/Program.cs | 6 +- .../Templates/Operator.CSharp/Program.cs | 6 +- .../Templates/WebOperator.CSharp/Program.cs | 6 +- .../Crds/CrdInstallerSettingsBuilder.Test.cs | 62 ++++++ .../Builder/OperatorBuilder.Test.cs | 19 ++ .../Crds/CrdInstaller.Test.cs | 149 +++++++++++++ .../Retry/ExponentialRetryBackoff.Test.cs | 30 +++ 20 files changed, 550 insertions(+), 83 deletions(-) create mode 100644 src/KubeOps.Abstractions/Crds/CrdInstallerSettingsBuilder.cs create mode 100644 src/KubeOps.Abstractions/Crds/CrdInstallerSettingsBuilderExtensions.cs create mode 100644 src/KubeOps.Operator/Retry/ExponentialRetryBackoff.cs create mode 100644 test/KubeOps.Abstractions.Test/Crds/CrdInstallerSettingsBuilder.Test.cs create mode 100644 test/KubeOps.Operator.Test/Crds/CrdInstaller.Test.cs create mode 100644 test/KubeOps.Operator.Test/Retry/ExponentialRetryBackoff.Test.cs diff --git a/docs/docs/operator/build-customization.mdx b/docs/docs/operator/build-customization.mdx index 1b12ac4f..e470386b 100644 --- a/docs/docs/operator/build-customization.mdx +++ b/docs/docs/operator/build-customization.mdx @@ -251,11 +251,9 @@ For example, in your `Program.cs`: builder.Services .AddKubernetesOperator() #if DEBUG - .AddCrdInstaller(c => - { - c.OverwriteExisting = true; - c.DeleteOnShutdown = true; - }) + .AddCrdInstaller(c => c + .WithOverwriteExisting() + .WithDeleteOnShutdown()) #endif .RegisterComponents(); ``` @@ -269,4 +267,4 @@ And in your project file: ``` -This setup ensures that resources are always generated and installed automatically during development, but you can disable or restrict this behavior for production builds as needed. \ No newline at end of file +This setup ensures that resources are always generated and installed automatically during development, but you can disable or restrict this behavior for production builds as needed. diff --git a/docs/docs/operator/utilities.mdx b/docs/docs/operator/utilities.mdx index f133af90..82b87db1 100644 --- a/docs/docs/operator/utilities.mdx +++ b/docs/docs/operator/utilities.mdx @@ -53,6 +53,7 @@ The CRD Installer is a powerful utility intended **only for development environm ::: When developing operators, you may want to quickly install or update CustomResourceDefinitions (CRDs) in your cluster. The `CrdInstaller` service automates this process, making it easier to iterate on CRD changes during development. +If the Kubernetes API server is temporarily unavailable when the operator starts, the installer logs the error and retries in the background with backoff instead of stopping the host startup. ### How to Add the CRD Installer @@ -62,14 +63,12 @@ To enable the CRD installer, add the following to your operator's `Program.cs`: builder.Services .AddKubernetesOperator() #if DEBUG - .AddCrdInstaller(c => - { - c.OverwriteExisting = true; - c.DeleteOnShutdown = true; - }) + .AddCrdInstaller(c => c + .WithOverwriteExisting() + .WithDeleteOnShutdown()) #endif .RegisterComponents(); ``` -- `OverwriteExisting`: If `true`, existing CRDs with the same name will be **overwritten**. This is useful for development but can be destructive if used in production, as it may cause data loss. -- `DeleteOnShutdown`: If `true`, all CRDs installed by the operator will be **deleted** when the operator shuts down. This is extremely destructive and should only be used in disposable development environments. \ No newline at end of file +- `WithOverwriteExisting()`: Existing CRDs with the same name will be **overwritten**. This is useful for development but can be destructive if used in production, as it may cause data loss. +- `WithDeleteOnShutdown()`: All CRDs installed by the operator will be **deleted** when the operator shuts down. This is extremely destructive and should only be used in disposable development environments. diff --git a/examples/Operator/Program.cs b/examples/Operator/Program.cs index 395080f0..2a994ac3 100644 --- a/examples/Operator/Program.cs +++ b/examples/Operator/Program.cs @@ -14,11 +14,9 @@ builder.Services .AddKubernetesOperator() #if DEBUG - .AddCrdInstaller(c => - { - c.OverwriteExisting = true; - c.DeleteOnShutdown = true; - }) + .AddCrdInstaller(c => c + .WithOverwriteExisting() + .WithDeleteOnShutdown()) #endif .RegisterComponents(); diff --git a/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs b/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs index 2cc630f2..b191daef 100644 --- a/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs +++ b/src/KubeOps.Abstractions/Builder/IOperatorBuilder.cs @@ -78,10 +78,10 @@ IOperatorBuilder AddFinalizer(string identifier) /// This is intended for development purposes only. /// /// - /// Configuration action for the . + /// Configuration action for the . /// Determines the behavior of the CRD installer, such as whether existing CRDs /// should be overwritten or deleted on shutdown. /// /// The builder for chaining. - IOperatorBuilder AddCrdInstaller(Action? configure = null); + IOperatorBuilder AddCrdInstaller(Action? configure = null); } diff --git a/src/KubeOps.Abstractions/Crds/CrdInstallerSettings.cs b/src/KubeOps.Abstractions/Crds/CrdInstallerSettings.cs index 8cb865cc..50a3450e 100644 --- a/src/KubeOps.Abstractions/Crds/CrdInstallerSettings.cs +++ b/src/KubeOps.Abstractions/Crds/CrdInstallerSettings.cs @@ -5,20 +5,20 @@ namespace KubeOps.Abstractions.Crds; /// -/// Settings for the CRD installer. +/// Immutable settings for the CRD installer. Created via . /// -public sealed class CrdInstallerSettings +public sealed record CrdInstallerSettings { /// /// Determines whether existing CRDs should be overwritten. /// This is useful for development purposes and should be used with caution. /// It is a destructive operation that may lead to data loss. /// - public bool OverwriteExisting { get; set; } = false; + public required bool OverwriteExisting { get; init; } /// /// Determines whether the installed CRDs should be deleted when the operator shuts down. /// This is a very destructive operation and should only be used in development environments. /// - public bool DeleteOnShutdown { get; set; } = false; + public required bool DeleteOnShutdown { get; init; } } diff --git a/src/KubeOps.Abstractions/Crds/CrdInstallerSettingsBuilder.cs b/src/KubeOps.Abstractions/Crds/CrdInstallerSettingsBuilder.cs new file mode 100644 index 00000000..7949f261 --- /dev/null +++ b/src/KubeOps.Abstractions/Crds/CrdInstallerSettingsBuilder.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace KubeOps.Abstractions.Crds; + +/// +/// Configures a instance. +/// Set properties directly or use the fluent With* extension methods, +/// then call to obtain the immutable record. +/// +public sealed class CrdInstallerSettingsBuilder +{ + /// + /// Determines whether existing CRDs should be overwritten. + /// This is useful for development purposes and should be used with caution. + /// It is a destructive operation that may lead to data loss. + /// + public bool OverwriteExisting { get; set; } + + /// + /// Determines whether the installed CRDs should be deleted when the operator shuts down. + /// This is a very destructive operation and should only be used in development environments. + /// + public bool DeleteOnShutdown { get; set; } + + /// + /// Produces an immutable record from the current configuration. + /// + /// A fully initialised record. + public CrdInstallerSettings Build() => new() + { + OverwriteExisting = OverwriteExisting, + DeleteOnShutdown = DeleteOnShutdown, + }; +} diff --git a/src/KubeOps.Abstractions/Crds/CrdInstallerSettingsBuilderExtensions.cs b/src/KubeOps.Abstractions/Crds/CrdInstallerSettingsBuilderExtensions.cs new file mode 100644 index 00000000..1068c9bd --- /dev/null +++ b/src/KubeOps.Abstractions/Crds/CrdInstallerSettingsBuilderExtensions.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace KubeOps.Abstractions.Crds; + +/// +/// Fluent extension methods for . +/// Each method sets one property and returns the same builder instance for chaining. +/// +public static class CrdInstallerSettingsBuilderExtensions +{ + /// Sets whether existing CRDs should be overwritten. + /// The builder to configure. + /// true to overwrite existing CRDs; false otherwise. + /// The same instance for chaining. + public static CrdInstallerSettingsBuilder WithOverwriteExisting( + this CrdInstallerSettingsBuilder builder, + bool value = true) + { + builder.OverwriteExisting = value; + return builder; + } + + /// Sets whether installed CRDs should be deleted when the operator shuts down. + /// The builder to configure. + /// true to delete installed CRDs on shutdown; false otherwise. + /// The same instance for chaining. + public static CrdInstallerSettingsBuilder WithDeleteOnShutdown( + this CrdInstallerSettingsBuilder builder, + bool value = true) + { + builder.DeleteOnShutdown = value; + return builder; + } +} diff --git a/src/KubeOps.Operator/Builder/OperatorBuilder.cs b/src/KubeOps.Operator/Builder/OperatorBuilder.cs index 825ab609..30302a2d 100644 --- a/src/KubeOps.Operator/Builder/OperatorBuilder.cs +++ b/src/KubeOps.Operator/Builder/OperatorBuilder.cs @@ -98,11 +98,12 @@ public IOperatorBuilder AddFinalizer(string identifier return this; } - public IOperatorBuilder AddCrdInstaller(Action? configure = null) + public IOperatorBuilder AddCrdInstaller(Action? configure = null) { - var settings = new CrdInstallerSettings(); - configure?.Invoke(settings); - Services.AddSingleton(settings); + var settingsBuilder = new CrdInstallerSettingsBuilder(); + configure?.Invoke(settingsBuilder); + + Services.AddSingleton(settingsBuilder.Build()); Services.AddHostedService(); return this; } diff --git a/src/KubeOps.Operator/Crds/CrdInstaller.cs b/src/KubeOps.Operator/Crds/CrdInstaller.cs index 3831aee7..34c70590 100644 --- a/src/KubeOps.Operator/Crds/CrdInstaller.cs +++ b/src/KubeOps.Operator/Crds/CrdInstaller.cs @@ -2,14 +2,17 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. +using System.Net; using System.Reflection; using System.Runtime.InteropServices; +using k8s; using k8s.Models; using KubeOps.Abstractions.Crds; using KubeOps.Abstractions.Entities.Attributes; using KubeOps.KubernetesClient; +using KubeOps.Operator.Retry; using KubeOps.Transpiler; using Microsoft.Extensions.Hosting; @@ -17,18 +20,167 @@ namespace KubeOps.Operator.Crds; -internal class CrdInstaller(ILogger logger, CrdInstallerSettings settings, IKubernetesClient client) - : IHostedService +internal sealed class CrdInstaller : IHostedService, IDisposable, IAsyncDisposable { + private readonly IKubernetesClient _client; + private readonly CancellationTokenSource _cts = new(); + private readonly Func _retryDelayFactory; + private readonly ILogger _logger; + private readonly CrdInstallerSettings _settings; private List _crds = []; + private bool _disposed; + private Task? _installationTask; - public async Task StartAsync(CancellationToken cancellationToken) + public CrdInstaller(ILogger logger, CrdInstallerSettings settings, IKubernetesClient client) + : this( + logger, + settings, + client, + ExponentialRetryBackoff.GetDelayWithJitter) { - logger.LogInformation("Execute CRD installer with overwrite: {Overwrite}", settings.OverwriteExisting); + } + + internal CrdInstaller( + ILogger logger, + CrdInstallerSettings settings, + IKubernetesClient client, + Func retryDelayFactory) + { + _logger = logger; + _settings = settings; + _client = client; + _retryDelayFactory = retryDelayFactory; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + _installationTask = Task.Run(RunInstallerWithRetriesAsync, CancellationToken.None); + return Task.CompletedTask; + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + if (_disposed) + { + return; + } + + await _cts.CancelAsync(); + + if (_installationTask is not null) + { + await _installationTask.WaitAsync(cancellationToken); + } + + if (!_settings.DeleteOnShutdown) + { + _logger.LogDebug("Skipping CRD deletion on shutdown as per settings."); + return; + } + + _logger.LogInformation("Deleting CRDs on shutdown."); + foreach (var crd in _crds) + { + try + { + _logger.LogInformation("Deleting CRD {Name}.", crd.Name()); + await _client.DeleteAsync(crd, cancellationToken); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to delete CRD {Name}.", crd.Name()); + } + } + } + + public void Dispose() + { + _cts.Dispose(); + _disposed = true; + } + + public async ValueTask DisposeAsync() + { + await CastAndDispose(_cts); + _disposed = true; + + static async ValueTask CastAndDispose(IDisposable resource) + { + if (resource is IAsyncDisposable resourceAsyncDisposable) + { + await resourceAsyncDisposable.DisposeAsync(); + } + else + { + resource.Dispose(); + } + } + } + + private async Task RunInstallerWithRetriesAsync() + { + uint installRetries = 0; + + while (!_cts.IsCancellationRequested) + { + try + { + await InstallAsync(_cts.Token); + return; + } + catch (OperationCanceledException) when (_cts.IsCancellationRequested) + { + return; + } + catch (Exception exception) when (IsTransient(exception)) + { + installRetries++; + var delay = _retryDelayFactory(installRetries); + + _logger.LogError( + exception, + "Failed to install CRDs. Wait {Seconds}s before attempting to install them again.", + delay.TotalSeconds); + try + { + await Task.Delay(delay, _cts.Token); + } + catch (OperationCanceledException) when (_cts.IsCancellationRequested) + { + return; + } + } + catch (Exception exception) + { + _logger.LogError(exception, "Failed to install CRDs due to a non-transient error."); + return; + } + } + + return; + + static bool IsTransient(Exception exception) + { + return exception switch + { + HttpRequestException or TimeoutException or TaskCanceledException => true, + KubernetesException { Status.Code: null } => true, + KubernetesException { Status.Code: (int)HttpStatusCode.RequestTimeout } => true, + KubernetesException { Status.Code: (int)HttpStatusCode.Conflict } => true, + KubernetesException { Status.Code: (int)HttpStatusCode.TooManyRequests } => true, + KubernetesException { Status.Code: >= 500 } => true, + _ => false, + }; + } + } + + private async Task InstallAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Execute CRD installer with overwrite: {Overwrite}", _settings.OverwriteExisting); var assembly = Assembly.GetEntryAssembly(); if (assembly is null) { - logger.LogError("No entry assembly found, cannot install CRDs."); + _logger.LogError("No entry assembly found, cannot install CRDs."); return; } @@ -52,45 +204,22 @@ public async Task StartAsync(CancellationToken cancellationToken) foreach (var crd in _crds) { var existing = - await client.GetAsync(crd.Name(), cancellationToken: cancellationToken); - if (existing is not null && !settings.OverwriteExisting) + await _client.GetAsync(crd.Name(), cancellationToken: cancellationToken); + if (existing is not null && !_settings.OverwriteExisting) { - logger.LogDebug("CRD {Name} already exists, skipping installation.", crd.Name()); + _logger.LogDebug("CRD {Name} already exists, skipping installation.", crd.Name()); } else if (existing is not null) { - logger.LogDebug("CRD {Name} already exists.", crd.Name()); - logger.LogInformation("Overwriting existing CRD {Name}.", crd.Name()); + _logger.LogDebug("CRD {Name} already exists.", crd.Name()); + _logger.LogInformation("Overwriting existing CRD {Name}.", crd.Name()); crd.Metadata.ResourceVersion = existing.ResourceVersion(); - await client.UpdateAsync(crd, cancellationToken); + await _client.UpdateAsync(crd, cancellationToken); } else { - logger.LogInformation("Installing CRD {Name}.", crd.Name()); - await client.CreateAsync(crd, cancellationToken); - } - } - } - - public async Task StopAsync(CancellationToken cancellationToken) - { - if (!settings.DeleteOnShutdown) - { - logger.LogDebug("Skipping CRD deletion on shutdown as per settings."); - return; - } - - logger.LogInformation("Deleting CRDs on shutdown."); - foreach (var crd in _crds) - { - try - { - logger.LogInformation("Deleting CRD {Name}.", crd.Name()); - await client.DeleteAsync(crd, cancellationToken); - } - catch (Exception ex) - { - logger.LogError(ex, "Failed to delete CRD {Name}.", crd.Name()); + _logger.LogInformation("Installing CRD {Name}.", crd.Name()); + await _client.CreateAsync(crd, cancellationToken); } } } diff --git a/src/KubeOps.Operator/LeaderElection/LeaderElectionBackgroundService.cs b/src/KubeOps.Operator/LeaderElection/LeaderElectionBackgroundService.cs index c2d2fe29..456d8ca3 100644 --- a/src/KubeOps.Operator/LeaderElection/LeaderElectionBackgroundService.cs +++ b/src/KubeOps.Operator/LeaderElection/LeaderElectionBackgroundService.cs @@ -4,6 +4,8 @@ using k8s.LeaderElection; +using KubeOps.Operator.Retry; + using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -98,10 +100,7 @@ private async Task RunAndTryToHoldLeadershipForeverAsync() { leadershipRetries++; - var delay = TimeSpan - .FromSeconds(Math.Pow(2, Math.Clamp(leadershipRetries, 0, 5))) - .Add(TimeSpan.FromMilliseconds(Random.Shared.Next(0, 1000))); - + var delay = ExponentialRetryBackoff.GetDelayWithJitter(leadershipRetries); logger.LogError(exception, "Failed to hold leadership. Wait {Seconds}s before attempting to reacquire leadership.", delay.TotalSeconds); await Task.Delay(delay); } diff --git a/src/KubeOps.Operator/Retry/ExponentialRetryBackoff.cs b/src/KubeOps.Operator/Retry/ExponentialRetryBackoff.cs new file mode 100644 index 00000000..c14a960c --- /dev/null +++ b/src/KubeOps.Operator/Retry/ExponentialRetryBackoff.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +namespace KubeOps.Operator.Retry; + +internal static class ExponentialRetryBackoff +{ + public static TimeSpan GetDelayWithJitter(uint retryCount) => TimeSpan + .FromSeconds(Math.Pow(2, Math.Clamp(retryCount, 0, 5))) + .Add(TimeSpan.FromMilliseconds(Random.Shared.Next(0, 1000))); +} diff --git a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs index d2d4010d..88e59601 100644 --- a/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs +++ b/src/KubeOps.Operator/Watcher/ResourceWatcher{TEntity}.cs @@ -18,6 +18,7 @@ using KubeOps.Operator.Logging; using KubeOps.Operator.Queue; using KubeOps.Operator.Reconciliation; +using KubeOps.Operator.Retry; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -339,9 +340,7 @@ e.InnerException is EndOfStreamException && logger.LogError(e, """There was an error while watching the resource "{Resource}".""", typeof(TEntity)); _watcherReconnectRetries++; - var delay = TimeSpan - .FromSeconds(Math.Pow(2, Math.Clamp(_watcherReconnectRetries, 0, 5))) - .Add(TimeSpan.FromMilliseconds(Random.Shared.Next(0, 1000))); + var delay = ExponentialRetryBackoff.GetDelayWithJitter(_watcherReconnectRetries); logger.LogWarning( "There were {Retries} errors / retries in the watcher. Wait {Seconds}s before next attempt to connect.", _watcherReconnectRetries, diff --git a/src/KubeOps.Templates/Templates/EmptyOperator.CSharp/Program.cs b/src/KubeOps.Templates/Templates/EmptyOperator.CSharp/Program.cs index 8e20b34e..7b6f2d69 100644 --- a/src/KubeOps.Templates/Templates/EmptyOperator.CSharp/Program.cs +++ b/src/KubeOps.Templates/Templates/EmptyOperator.CSharp/Program.cs @@ -10,9 +10,9 @@ #if DEBUG .AddCrdInstaller(c => { - // Careful, this can be very destructive. - // c.OverwriteExisting = true; - // c.DeleteOnShutdown = true; + // Careful, these can be very destructive. + // c.WithOverwriteExisting() + // .WithDeleteOnShutdown(); }) #endif //+:cnd:noEmit diff --git a/src/KubeOps.Templates/Templates/EmptyWebOperator.CSharp/Program.cs b/src/KubeOps.Templates/Templates/EmptyWebOperator.CSharp/Program.cs index e716b664..cf177e8f 100644 --- a/src/KubeOps.Templates/Templates/EmptyWebOperator.CSharp/Program.cs +++ b/src/KubeOps.Templates/Templates/EmptyWebOperator.CSharp/Program.cs @@ -11,9 +11,9 @@ #if DEBUG .AddCrdInstaller(c => { - // Careful, this can be very destructive. - // c.OverwriteExisting = true; - // c.DeleteOnShutdown = true; + // Careful, these can be very destructive. + // c.WithOverwriteExisting() + // .WithDeleteOnShutdown(); }) #endif //+:cnd:noEmit diff --git a/src/KubeOps.Templates/Templates/Operator.CSharp/Program.cs b/src/KubeOps.Templates/Templates/Operator.CSharp/Program.cs index 8e20b34e..7b6f2d69 100644 --- a/src/KubeOps.Templates/Templates/Operator.CSharp/Program.cs +++ b/src/KubeOps.Templates/Templates/Operator.CSharp/Program.cs @@ -10,9 +10,9 @@ #if DEBUG .AddCrdInstaller(c => { - // Careful, this can be very destructive. - // c.OverwriteExisting = true; - // c.DeleteOnShutdown = true; + // Careful, these can be very destructive. + // c.WithOverwriteExisting() + // .WithDeleteOnShutdown(); }) #endif //+:cnd:noEmit diff --git a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Program.cs b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Program.cs index e716b664..cf177e8f 100644 --- a/src/KubeOps.Templates/Templates/WebOperator.CSharp/Program.cs +++ b/src/KubeOps.Templates/Templates/WebOperator.CSharp/Program.cs @@ -11,9 +11,9 @@ #if DEBUG .AddCrdInstaller(c => { - // Careful, this can be very destructive. - // c.OverwriteExisting = true; - // c.DeleteOnShutdown = true; + // Careful, these can be very destructive. + // c.WithOverwriteExisting() + // .WithDeleteOnShutdown(); }) #endif //+:cnd:noEmit diff --git a/test/KubeOps.Abstractions.Test/Crds/CrdInstallerSettingsBuilder.Test.cs b/test/KubeOps.Abstractions.Test/Crds/CrdInstallerSettingsBuilder.Test.cs new file mode 100644 index 00000000..0d9b5c7c --- /dev/null +++ b/test/KubeOps.Abstractions.Test/Crds/CrdInstallerSettingsBuilder.Test.cs @@ -0,0 +1,62 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using KubeOps.Abstractions.Crds; + +namespace KubeOps.Abstractions.Test.Crds; + +public sealed class CrdInstallerSettingsBuilderTest +{ + [Fact] + public void Build_Produces_Correct_Default_Values() + { + var settings = new CrdInstallerSettingsBuilder().Build(); + + settings.OverwriteExisting.Should().BeFalse(); + settings.DeleteOnShutdown.Should().BeFalse(); + } + + [Fact] + public void Builder_Accepts_All_Property_Setters_And_Passes_Them_Through_Build() + { + var settings = new CrdInstallerSettingsBuilder + { + OverwriteExisting = true, + DeleteOnShutdown = true, + }.Build(); + + settings.OverwriteExisting.Should().BeTrue(); + settings.DeleteOnShutdown.Should().BeTrue(); + } + + [Fact] + public void Fluent_Api_Sets_All_Properties_And_Builds_Correctly() + { + var settings = new CrdInstallerSettingsBuilder() + .WithOverwriteExisting() + .WithDeleteOnShutdown() + .Build(); + + settings.OverwriteExisting.Should().BeTrue(); + settings.DeleteOnShutdown.Should().BeTrue(); + } + + [Fact] + public void Fluent_Api_Can_Disable_All_Properties_Explicitly() + { + var settings = new CrdInstallerSettingsBuilder + { + OverwriteExisting = true, + DeleteOnShutdown = true, + } + .WithOverwriteExisting(false) + .WithDeleteOnShutdown(false) + .Build(); + + settings.OverwriteExisting.Should().BeFalse(); + settings.DeleteOnShutdown.Should().BeFalse(); + } +} diff --git a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs index 3028ed24..0559a796 100644 --- a/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs +++ b/test/KubeOps.Operator.Test/Builder/OperatorBuilder.Test.cs @@ -5,6 +5,7 @@ using FluentAssertions; using KubeOps.Abstractions.Builder; +using KubeOps.Abstractions.Crds; using KubeOps.Abstractions.Entities; using KubeOps.Abstractions.Events; using KubeOps.Abstractions.Reconciliation; @@ -143,6 +144,24 @@ public void Should_Add_LeaderAwareResourceWatcher() s.Lifetime == ServiceLifetime.Singleton); } + [Fact] + public void Should_Add_CrdInstaller_Settings() + { + _builder.AddCrdInstaller(c => c + .WithOverwriteExisting() + .WithDeleteOnShutdown()); + + var settingsDescriptor = _builder.Services.Single(s => s.ServiceType == typeof(CrdInstallerSettings)); + + settingsDescriptor.ImplementationInstance.Should().BeEquivalentTo(new + { + OverwriteExisting = true, + DeleteOnShutdown = true, + }); + settingsDescriptor.Lifetime.Should().Be(ServiceLifetime.Singleton); + _builder.Services.Should().NotContain(s => s.ServiceType == typeof(CrdInstallerSettingsBuilder)); + } + private sealed class TestController : IEntityController { public Task> ReconcileAsync(V1OperatorIntegrationTestEntity entity, CancellationToken cancellationToken) => diff --git a/test/KubeOps.Operator.Test/Crds/CrdInstaller.Test.cs b/test/KubeOps.Operator.Test/Crds/CrdInstaller.Test.cs new file mode 100644 index 00000000..c42878f6 --- /dev/null +++ b/test/KubeOps.Operator.Test/Crds/CrdInstaller.Test.cs @@ -0,0 +1,149 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using System.Net; + +using FluentAssertions; + +using k8s; +using k8s.Models; + +using KubeOps.Abstractions.Crds; +using KubeOps.KubernetesClient; + +using Microsoft.Extensions.Logging; + +using Moq; + +using OperatorCrdInstaller = KubeOps.Operator.Crds.CrdInstaller; + +namespace KubeOps.Operator.Test.Crds; + +public sealed class CrdInstallerTest +{ + [Fact] + public async Task StartAsync_Should_Not_Propagate_Transient_Error() + { + var clientMock = CreateClientMock(); + clientMock + .Setup(c => c.GetAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ThrowsAsync(new HttpRequestException("API server unavailable")); + + var installer = CreateInstaller(clientMock.Object); + + Func action = () => installer.StartAsync(TestContext.Current.CancellationToken); + + await action.Should().NotThrowAsync(); + await installer.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task Transient_Error_Should_Be_Retried() + { + var clientMock = CreateClientMock(); + using var secondAttempt = new ManualResetEventSlim(false); + var attempts = 0; + + clientMock + .Setup(c => c.GetAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns( + (_, _, _) => + { + attempts++; + if (attempts == 1) + { + throw new HttpRequestException("API server unavailable"); + } + + secondAttempt.Set(); + return Task.FromResult(null); + }); + + var installer = CreateInstaller(clientMock.Object); + + await installer.StartAsync(TestContext.Current.CancellationToken); + + SpinWait.SpinUntil(() => secondAttempt.IsSet, TimeSpan.FromSeconds(1)).Should().BeTrue(); + await installer.StopAsync(TestContext.Current.CancellationToken); + } + + [Fact] + public async Task Non_Transient_Kubernetes_Error_Should_Not_Be_Retried() + { + var clientMock = CreateClientMock(); + using var firstAttempt = new ManualResetEventSlim(false); + + clientMock + .Setup(c => c.GetAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback(() => firstAttempt.Set()) + .ThrowsAsync(new KubernetesException(new V1Status { Code = (int)HttpStatusCode.Forbidden })); + + var installer = CreateInstaller(clientMock.Object); + + await installer.StartAsync(TestContext.Current.CancellationToken); + + SpinWait.SpinUntil(() => firstAttempt.IsSet, TimeSpan.FromSeconds(1)).Should().BeTrue(); + await Task.Delay(TimeSpan.FromMilliseconds(100), TestContext.Current.CancellationToken); + await installer.StopAsync(TestContext.Current.CancellationToken); + + clientMock.Verify( + c => c.GetAsync( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task StopAsync_Should_Cancel_Pending_Retry_Delay() + { + var clientMock = CreateClientMock(); + using var firstAttempt = new ManualResetEventSlim(false); + + clientMock + .Setup(c => c.GetAsync( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback(() => firstAttempt.Set()) + .ThrowsAsync(new HttpRequestException("API server unavailable")); + + var installer = CreateInstaller(clientMock.Object, _ => TimeSpan.FromSeconds(10)); + + await installer.StartAsync(TestContext.Current.CancellationToken); + + SpinWait.SpinUntil(() => firstAttempt.IsSet, TimeSpan.FromSeconds(1)).Should().BeTrue(); + var stopTask = installer.StopAsync(TestContext.Current.CancellationToken); + + await stopTask.WaitAsync(TimeSpan.FromSeconds(1), TestContext.Current.CancellationToken); + } + + private static OperatorCrdInstaller CreateInstaller( + IKubernetesClient client, + Func? retryDelayFactory = null) + => new( + Mock.Of>(), + new CrdInstallerSettingsBuilder().Build(), + client, + retryDelayFactory ?? (_ => TimeSpan.Zero)); + + private static Mock CreateClientMock() + { + var clientMock = new Mock(); + clientMock + .Setup(c => c.CreateAsync(It.IsAny(), It.IsAny())) + .Returns((crd, _) => Task.FromResult(crd)); + + return clientMock; + } +} diff --git a/test/KubeOps.Operator.Test/Retry/ExponentialRetryBackoff.Test.cs b/test/KubeOps.Operator.Test/Retry/ExponentialRetryBackoff.Test.cs new file mode 100644 index 00000000..fb002736 --- /dev/null +++ b/test/KubeOps.Operator.Test/Retry/ExponentialRetryBackoff.Test.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +using FluentAssertions; + +using KubeOps.Operator.Retry; + +namespace KubeOps.Operator.Test.Retry; + +public sealed class ExponentialRetryBackoffTest +{ + [Fact] + public void GetDelayWithJitter_Should_Use_Exponential_Backoff() + { + var delay = ExponentialRetryBackoff.GetDelayWithJitter(1); + + delay.Should().BeGreaterThanOrEqualTo(TimeSpan.FromSeconds(2)); + delay.Should().BeLessThan(TimeSpan.FromSeconds(3)); + } + + [Fact] + public void GetDelayWithJitter_Should_Cap_Exponential_Backoff() + { + var delay = ExponentialRetryBackoff.GetDelayWithJitter(10); + + delay.Should().BeGreaterThanOrEqualTo(TimeSpan.FromSeconds(32)); + delay.Should().BeLessThan(TimeSpan.FromSeconds(33)); + } +}