Merge branch 'develop' into feature/dotnet-10

Author: hhvrc

.github/actions/kubernetes-rollout-restart/action.yml

@@ -51,7 +51,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - uses: actions/setup-dotnet@v5
         with:
@@ -78,7 +78,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -114,7 +114,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Log in to Container Registry
         uses: docker/login-action@v3
@@ -172,7 +172,7 @@ jobs:
     environment: development
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           sparse-checkout: |
             .github

.github/workflows/ci-build.yml

@@ -27,7 +27,7 @@ jobs:
       contents: read
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v5
+      uses: actions/checkout@v6
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/codeql.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           ref: ${{ github.ref }}
 

.github/workflows/update-cloudflare-proxies.yml

@@ -56,13 +56,42 @@ public async Task StartAsync(CancellationToken cancellationToken)
 
     private void HandleKeyExpired(RedisChannel _, RedisValue message)
     {
-        if (!message.HasValue) return;
-        if (message.ToString().Split(':', 2) is not [{ } guid, { } name]) return;
-
-        if (!Guid.TryParse(guid, out var id)) return;
-
-        if (typeof(DeviceOnline).FullName == name)
+        if (!message.HasValue)
+        {
+            _logger.LogWarning("Received expired key with empty value for hub offline status");
+            return;
+        }
+        
+        var messageString = (string?)message;
+        if (messageString is null)
+        {
+            _logger.LogWarning("Received expired key that could not be converted to string for hub offline status. Raw value type: {ValueType}", message.GetType().FullName);
+            return;
+        }
+        
+        var messageSpan = messageString.AsSpan();
+        
+        // We always expect TypeName:GUID right now, if GUID is not present, something is really wrong
+        var colonPos = messageSpan.IndexOf(':');
+        if (colonPos < 0)
         {
+            _logger.LogError("Received expired key with unexpected format (missing colon) for hub offline status. Value: {MessageValue}", messageString);
+            return;
+        }
+        
+        // Data structure is TypeName:GUID
+        var typeNameSpan = messageSpan[..colonPos];
+        var guidSpan = messageSpan[(colonPos + 1)..];
+        
+        // Check what type of expired key this is
+        if (typeNameSpan.SequenceEqual(typeof(DeviceOnline).FullName))
+        {
+            if (!Guid.TryParse(guidSpan, out var id))
+            {
+                _logger.LogError("Received expired key with invalid GUID for hub offline status: {MessageValue}", messageString);
+                return;
+            }
+            
             OsTask.Run(() => LogicDeviceOnlineStatus(id));
         }
     }

API/Realtime/RedisSubscriberService.cs

@@ -38,7 +38,8 @@ public static IServiceCollection AddOpenShockMemDB(this IServiceCollection servi
     {
         // <---- Redis ---->
         services.AddSingleton<IConnectionMultiplexer>(ConnectionMultiplexer.Connect(options));
-        services.AddSingleton<IRedisConnectionProvider, RedisConnectionProvider>(serviceProvider => new RedisConnectionProvider(serviceProvider.GetRequiredService<IConnectionMultiplexer>()));
+        services.AddSingleton<IRedisConnectionProvider, RedisConnectionProvider>(serviceProvider =>
+            new RedisConnectionProvider(serviceProvider.GetRequiredService<IConnectionMultiplexer>()));
         services.AddSingleton<IRedisPubService, RedisPubService>();
 
         return services;
@@ -47,7 +48,8 @@ public static IServiceCollection AddOpenShockMemDB(this IServiceCollection servi
     public static IServiceCollection AddOpenShockDB(this IServiceCollection services, DatabaseOptions options)
     {
         // <---- Postgres EF Core ---->
-        services.AddDbContextPool<OpenShockContext>(builder => OpenShockContext.ConfigureOptionsBuilder(builder, options.Conn, options.Debug));
+        services.AddDbContextPool<OpenShockContext>(builder =>
+            OpenShockContext.ConfigureOptionsBuilder(builder, options.Conn, options.Debug));
         services.AddPooledDbContextFactory<OpenShockContext>(builder =>
         {
             builder.UseNpgsql(options.Conn);
@@ -67,7 +69,8 @@ public static IServiceCollection AddOpenShockDB(this IServiceCollection services
     /// <param name="services"></param>
     /// <param name="configureOptions"></param>
     /// <returns></returns>
-    private static AuthenticationBuilder AddOpenShockAuthentication(this IServiceCollection services, Action<AuthenticationOptions> configureOptions)
+    private static AuthenticationBuilder AddOpenShockAuthentication(this IServiceCollection services,
+        Action<AuthenticationOptions> configureOptions)
     {
         ArgumentNullException.ThrowIfNull(services);
         ArgumentNullException.ThrowIfNull(configureOptions);
@@ -78,9 +81,9 @@ private static AuthenticationBuilder AddOpenShockAuthentication(this IServiceCol
         services.TryAddSingleton(TimeProvider.System);
         // services.TryAddSingleton<ISystemClock, SystemClock>(); // Exists in original AddAuthentication method
         // services.TryAddSingleton<IAuthenticationConfigurationProvider, DefaultAuthenticationConfigurationProvider>(); // Exists in original AddAuthentication method
-        
+
         var builder = new AuthenticationBuilder(services);
-        
+
         services.Configure(configureOptions);
 
         return builder;
@@ -91,8 +94,10 @@ private static AuthenticationBuilder AddOpenShockAuthentication(this IServiceCol
     /// </summary>
     /// <param name="services"></param>
     /// <param name="configureAuth"></param>
+    /// <param name="configureMetrics"></param>
     /// <returns></returns>
-    public static IServiceCollection AddOpenShockServices(this IServiceCollection services, Action<AuthenticationBuilder>? configureAuth = null)
+    public static IServiceCollection AddOpenShockServices(this IServiceCollection services,
+        Action<AuthenticationBuilder>? configureAuth = null, Action<MeterProviderBuilder>? configureMetrics = null)
     {
         // <---- ASP.NET ---->
         services.AddExceptionHandler<OpenShockExceptionHandler>();
@@ -116,7 +121,8 @@ public static IServiceCollection AddOpenShockServices(this IServiceCollection se
                 opt.DefaultScheme = OpenShockAuthSchemes.UserSessionCookie;
                 opt.DefaultAuthenticateScheme = OpenShockAuthSchemes.UserSessionCookie;
             })
-            .AddScheme<AuthenticationSchemeOptions, UserSessionAuthentication>(OpenShockAuthSchemes.UserSessionCookie, _ => { })
+            .AddScheme<AuthenticationSchemeOptions, UserSessionAuthentication>(OpenShockAuthSchemes.UserSessionCookie,
+                _ => { })
             .AddScheme<AuthenticationSchemeOptions, ApiTokenAuthentication>(OpenShockAuthSchemes.ApiToken, _ => { })
             .AddScheme<AuthenticationSchemeOptions, HubAuthentication>(OpenShockAuthSchemes.HubToken, _ => { });
 
@@ -127,12 +133,12 @@ public static IServiceCollection AddOpenShockServices(this IServiceCollection se
             options.AddPolicy(OpenShockAuthPolicies.RankAdmin, policy => policy.RequireRole("Admin", "System"));
             // TODO: Add token permission policies
         });
-        
+
         services.AddSingleton<IAuthorizationMiddlewareResultHandler, OpenShockAuthorizationMiddlewareResultHandler>();
-        
+
         services.ConfigureHttpJsonOptions(opt => JsonOptions.ConfigureDefault(opt.SerializerOptions));
         services.AddControllers().AddJsonOptions(opt => JsonOptions.ConfigureDefault(opt.JsonSerializerOptions));
-        
+
         var apiVersioningBuilder = services.AddApiVersioning(options =>
         {
             options.DefaultApiVersion = new ApiVersion(1, 0);
@@ -146,13 +152,13 @@ public static IServiceCollection AddOpenShockServices(this IServiceCollection se
             setup.DefaultApiVersion = new ApiVersion(1, 0);
             setup.AssumeDefaultVersionWhenUnspecified = true;
         });
-        
+
         // generic ASP.NET stuff
         services.AddMemoryCache();
         services.AddHttpContextAccessor();
         services.AddWebEncoders();
         services.AddProblemDetails();
-        
+
         services.AddCors(options =>
         {
             options.AddDefaultPolicy(builder =>
@@ -164,7 +170,7 @@ public static IServiceCollection AddOpenShockServices(this IServiceCollection se
                 builder.SetPreflightMaxAge(TimeSpan.FromHours(24));
             });
         });
-        
+
         // This needs to be at this position, earlier will break validation error responses
         services.Configure<ApiBehaviorOptions>(options =>
         {
@@ -174,16 +180,21 @@ public static IServiceCollection AddOpenShockServices(this IServiceCollection se
                 return problemDetails.ToObjectResult(context.HttpContext);
             };
         });
-        
+
         // OpenTelemetry
 
         services.AddOpenTelemetry()
-            .WithMetrics(metrics => metrics
-                .AddRuntimeInstrumentation()
-                .AddAspNetCoreInstrumentation()
-                .AddHttpClientInstrumentation()
-                .AddPrometheusExporter());
-        
+            .WithMetrics(metrics =>
+            {
+                metrics
+                    .AddRuntimeInstrumentation()
+                    .AddAspNetCoreInstrumentation()
+                    .AddHttpClientInstrumentation()
+                    .AddPrometheusExporter();
+
+                configureMetrics?.Invoke(metrics);
+            });
+
         // <---- OpenShock Services ---->
 
         services.AddScoped<IConfigurationService, ConfigurationService>();
@@ -233,16 +244,17 @@ await context.HttpContext.Response.WriteAsync("Too Many Requests. Please try aga
                     var ip = context.GetRemoteIP();
                     if (IPAddress.IsLoopback(ip)) return RateLimitPartition.GetNoLimiter("ip-loopback-nolimit");
 
-                    return RateLimitPartition.GetSlidingWindowLimiter($"ip-{ip}", _ => new SlidingWindowRateLimiterOptions
-                    {
-                        PermitLimit = 1000,
-                        Window = TimeSpan.FromMinutes(1),
-                        SegmentsPerWindow = 6,
-                        QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
-                        QueueLimit = 100
-                    });
+                    return RateLimitPartition.GetSlidingWindowLimiter($"ip-{ip}", _ =>
+                        new SlidingWindowRateLimiterOptions
+                        {
+                            PermitLimit = 1000,
+                            Window = TimeSpan.FromMinutes(1),
+                            SegmentsPerWindow = 6,
+                            QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
+                            QueueLimit = 100
+                        });
                 }
-                
+
                 if (user.HasClaim(claim => claim is { Type: ClaimTypes.Role, Value: "Admin" or "System" }))
                     return RateLimitPartition.GetNoLimiter("privileged-nolimit");
 
@@ -290,7 +302,8 @@ await context.HttpContext.Response.WriteAsync("Too Many Requests. Please try aga
         return services;
     }
 
-    public static IServiceCollection AddOpenShockSignalR(this IServiceCollection services, ConfigurationOptions redisConfig)
+    public static IServiceCollection AddOpenShockSignalR(this IServiceCollection services,
+        ConfigurationOptions redisConfig)
     {
         services.AddSignalR()
             .AddOpenShockStackExchangeRedis(options => { options.Configuration = redisConfig; })
@@ -299,7 +312,7 @@ public static IServiceCollection AddOpenShockSignalR(this IServiceCollection ser
                 options.PayloadSerializerOptions.PropertyNameCaseInsensitive = true;
                 options.PayloadSerializerOptions.Converters.Add(new SemVersionJsonConverter());
             });
-        
+
         return services;
     }
 }

Common/OpenShockServiceHelper.cs

@@ -2,7 +2,7 @@
   <PropertyGroup>
     <Company>OpenShock</Company>
     <Product>$(Company).$(MSBuildProjectName)</Product>
-    <Version>3.15.1</Version>
+    <Version>3.15.2</Version>
 
     <Title>$(Product)</Title>
     <Authors>OpenShock</Authors>

Directory.Build.props

@@ -15,7 +15,10 @@ public sealed class LcgKeepAlive : IHostedService
     private readonly LcgOptions _options;
     private readonly ILogger<LcgKeepAlive> _logger;
 
-    private const uint KeepAliveInterval = 35; // 35 seconds
+    private uint _errorsInRow;
+    
+    private static readonly TimeSpan KeepAliveKeyTTL = TimeSpan.FromSeconds(35); // 35 seconds
+    private static readonly TimeSpan KeepAliveInterval = TimeSpan.FromSeconds(15); // 15 seconds
 
     /// <summary>
     /// DI Constructor
@@ -45,7 +48,7 @@ await lcgNodes.InsertAsync(new LcgNode
                 Country = _options.CountryCode,
                 Load = 0,
                 Environment = _env.EnvironmentName
-            }, TimeSpan.FromSeconds(35));
+            }, KeepAliveKeyTTL);
             return;
         }
 
@@ -65,7 +68,7 @@ await lcgNodes.InsertAsync(new LcgNode
         }
 
         await _redisConnectionProvider.Connection.ExecuteAsync("EXPIRE",
-            $"{typeof(LcgNode).FullName}:{_options.Fqdn}", KeepAliveInterval);
+            $"{typeof(LcgNode).FullName}:{_options.Fqdn}", (int)KeepAliveKeyTTL.TotalSeconds);
     }
 
     private async Task Loop()
@@ -76,11 +79,21 @@ private async Task Loop()
             {
                 _logger.LogDebug("Sending keep alive...");
                 await SelfOnline();
-                await Task.Delay(15_000);
+                _logger.LogDebug("Sent keep alive!");
+                _errorsInRow = 0;
+                await Task.Delay(KeepAliveInterval);
             }
             catch (Exception e)
             {
-                _logger.LogError(e, "Error in loop");
+                ++_errorsInRow;
+                _logger.LogError(e, "Error sending gateway keep alive {Attempt}", _errorsInRow);
+                if(_errorsInRow >= 10)
+                {
+                    _logger.LogCritical("Too many errors in a row sending keep alive, terminating process");
+                    Environment.Exit(1001);
+                }
+                
+                await Task.Delay(KeepAliveInterval);
             }
         }
         // ReSharper disable once FunctionNeverReturns