Release/v3.2.0

.github/workflows/build-jhoose-security-cms13.yml

@@ -8,8 +8,8 @@ on:
   workflow_dispatch:
 
 env:
-  BUILD_NO: 3.1.0.${{ github.run_number }}  
-  BUILD_NO_PRE: 3.1.0-rc.${{ github.run_number }} 
+  BUILD_NO: 3.2.0.${{ github.run_number }}  
+  BUILD_NO_PRE: 3.2.0-rc.${{ github.run_number }} 
 
 jobs:
   build:

.github/workflows/build-jhoose-security.yml

@@ -8,8 +8,8 @@ on:
   workflow_dispatch:
 
 env:
-  BUILD_NO: 3.1.0.${{ github.run_number }}  
-  BUILD_NO_PRE: 3.1.0-rc.${{ github.run_number }} 
+  BUILD_NO: 3.2.0.${{ github.run_number }}  
+  BUILD_NO_PRE: 3.2.0-rc.${{ github.run_number }} 
 
 jobs:
   build:

README.md

@@ -305,5 +305,16 @@ X-API-Key: ...
  |2.6.2|Fixed another bug with (CRLF in header values)|
  |2.6.3|Fixed performance issues with the reporting API.<br/>Fixed a race condition that caused the nonce to leak across requests under high load.|
  |3.0.0|Added multisite support, CSP and Permissions Policy, Security Headers can now be configured per site.  |
- |3.0.4|Fixed issue with the files not being copied to the output directory when building the project, this was causing the module to not work when installed from NuGet.|
- |3.1.0| Added CMS13 Support.|
+ |3.0.4 |Fixed issue with the files not being copied to the output directory when building the project, this was causing the module to not work when installed from NuGet.|
+ |3.1.0 | Added CMS13 Support.|
+ |3.2.0 | Updated Purge scheduled job to run in batches : thanks @kennygutierrez
+
+
+  ---
+ ## Contributors
+
+https://github.com/Doom-83
+https://github.com/neorth
+https://github.com/kennygutierrez
+
+Thanks for all the support, suggestions, features and bugfixes

src/Jhoose.Security/Configuration/ReportingOptions.cs

@@ -6,6 +6,13 @@ public class ReportingOptions
 {
     public int RetainDays { get; set; } = 30;
 
+    /// <summary>
+    /// Maximum rows the purge job deletes per batch. Set to 0 to disable batching
+    /// (issue a single unbounded DELETE, matching pre-batching behavior).
+    /// Only the SQL provider honors this; ElasticSearch handles bulk deletes natively.
+    /// </summary>
+    public int PurgeBatchSize { get; set; } = 5000;
+
     public string UseProvider { get; set; } = string.Empty;
 
     public string ConnectionString { get; set; } = string.Empty;

src/Jhoose.Security/Features/Reporting/Database/SqlDatabaseReportingRepository.cs

@@ -100,13 +100,27 @@ await sqlHelper.ExecuteStoredProcedure("GetSecurityReportSummary", parameters, (
         }
     }
 
-    public async Task<int> PurgeReporingData(DateTime beforeDate)
+    public async Task<int> PurgeReporingData(DateTime beforeDate, int? batchSize = null)
     {
         try {
-            var sqlCommand = "DELETE FROM SecurityReportTo WHERE RecievedAt < @BeforeDate";
+            // Unbatched path preserves prior behavior for callers that pass no batch size.
+            if (!batchSize.HasValue || batchSize.Value <= 0)
+            {
+                var sqlCommand = "DELETE FROM SecurityReportTo WHERE RecievedAt < @BeforeDate";
+
+                return await sqlHelper.ExecuteNonQuery(
+                    sqlCommand,
+                    sqlHelper.CreateParameter<DateTime>("BeforeDate", SqlDbType.DateTime, beforeDate));
+            }
+
+            // Single batch: caller is expected to loop until rows == 0. Keeps each
+            // statement's transaction small enough to commit before the SqlCommand
+            // timeout, instead of rolling back a multi-hour DELETE.
+            var batchedSql = "DELETE TOP (@BatchSize) FROM SecurityReportTo WHERE RecievedAt < @BeforeDate";
 
             return await sqlHelper.ExecuteNonQuery(
-                sqlCommand,
+                batchedSql,
+                sqlHelper.CreateParameter<int>("BatchSize", SqlDbType.Int, batchSize.Value),
                 sqlHelper.CreateParameter<DateTime>("BeforeDate", SqlDbType.DateTime, beforeDate));
         }
         catch (Exception ex)

src/Jhoose.Security/Features/Reporting/ElasticSearch/ElasticSearchReportingRepository.cs

@@ -119,8 +119,13 @@ public async Task<DashboardSummary> GetDashboardSummary(DashboardSummary summary
         return summary;
     }
 
-    public async Task<int> PurgeReporingData(DateTime beforeDate)
+    public async Task<int> PurgeReporingData(DateTime beforeDate, int? batchSize = null)
     {
+        // batchSize is ignored: Elasticsearch DeleteByQuery handles large deletions
+        // server-side without the rollback risk SQL has, so the caller's batching
+        // loop is unnecessary here. The parameter exists only to satisfy the interface.
+        _ = batchSize;
+
         var response = await this.client.Value.DeleteByQueryAsync<ReportTo<IReportToBody>>(d => d.Query(query => query.Bool(b => b.Must(m => m.Range(r => r.DateRange(dr => dr.Field(f => f.RecievedAt).Gte(beforeDate)))))));
 
         var deletedCount = response.Deleted ?? 0;

src/Jhoose.Security/Features/Reporting/IReportingRepository.cs

@@ -18,7 +18,15 @@ public interface IReportingRepository
 
     Task<DashboardSummary> GetDashboardSummary(DashboardSummary summary);
 
-    Task<int> PurgeReporingData(DateTime beforeDate);
+    /// <summary>
+    /// Deletes reporting rows older than <paramref name="beforeDate"/>.
+    /// When <paramref name="batchSize"/> is provided and greater than zero, providers
+    /// that support it should delete at most that many rows per call so callers can
+    /// loop and bound the work each query does (avoiding command-timeout rollbacks on
+    /// large tables). When null, the call behaves as before and deletes everything in
+    /// one statement.
+    /// </summary>
+    Task<int> PurgeReporingData(DateTime beforeDate, int? batchSize = null);
 
     Task<CspSearchResults> Search(CspSearchParams searchParams);
 }

src/Jhoose.Security/Features/Reporting/Jobs/PurgeReporintgDataJob.cs

@@ -10,12 +10,13 @@
 
 namespace Jhoose.Security.Features.Reporting.Jobs;
 
 [ScheduledPlugIn(DisplayName = "Purge Jhoose Security Reporting Data", Description = "Purge old reporting data")]
 public class PurgeReporintgDataJob : ScheduledJobBase
 {
     private readonly IReportingRepositoryFactory reportingRepositoryFactory;
     private readonly IOptions<ReportingOptions> options;
     private readonly ILogger<PurgeReporintgDataJob> logger;
+    private bool stopSignaled;
 
     public PurgeReporintgDataJob(IReportingRepositoryFactory reportingRepositoryFactory,
                                  IOptions<ReportingOptions> options,
@@ -24,8 +25,11 @@
         this.reportingRepositoryFactory = reportingRepositoryFactory;
         this.options = options;
         this.logger = logger;
+        this.IsStoppable = true;
     }
 
+    public override void Stop() => stopSignaled = true;
+
     public override string Execute()
     {
         var reportingRepository = reportingRepositoryFactory.GetReportingRepository();
@@ -42,9 +46,37 @@
             }
 
             var beforeDate = DateTime.UtcNow.AddDays(options.Value.RetainDays * -1);
-            var purged = reportingRepository.PurgeReporingData(beforeDate).Result;
+            var batchSize = options.Value.PurgeBatchSize;
+
+            // PurgeBatchSize <= 0 disables batching and preserves the original
+            // single-DELETE behavior for anyone who relied on it.
+            if (batchSize <= 0)
+            {
+                var purgedOnce = reportingRepository.PurgeReporingData(beforeDate).Result;
+                return $"Purged {purgedOnce} records, from before {beforeDate}";
+            }
+
+            var totalPurged = 0;
+            var batches = 0;
+
+            OnStatusChanged($"Purging reporting rows older than {beforeDate:u} in batches of {batchSize}...");
+
+            while (!stopSignaled)
+            {
+                var purgedInBatch = reportingRepository.PurgeReporingData(beforeDate, batchSize).Result;
+                if (purgedInBatch <= 0)
+                {
+                    break;
+                }
+
+                totalPurged += purgedInBatch;
+                batches++;
+                OnStatusChanged($"Batch {batches}: purged {purgedInBatch} (total {totalPurged}).");
+            }
 
-            return $"Purged {purged} records, from before {beforeDate}";
+            return stopSignaled
+                ? $"Stopped. Purged {totalPurged} records, from before {beforeDate} across {batches} batches."
+                : $"Purged {totalPurged} records, from before {beforeDate} across {batches} batches.";
         }
         catch (Exception ex)
         {

src/Jhoose.Security/Jhoose.Security.csproj

@@ -36,7 +36,7 @@
 		<RepositoryUrl>https://github.com/andrewmarkham/contentsecuritypolicy</RepositoryUrl>
 		<ProjectUrl>https://github.com/andrewmarkham/contentsecuritypolicy</ProjectUrl>
 		<PackageReadmeFile>README.md</PackageReadmeFile>
-		<Version>3.1.0.0</Version>
+		<Version>3.2.0.0</Version>
 		<Authors>Andrew Markham</Authors>
 		<Description>Interface to manage Content Security Policy, Permissions Policy and OWASP Recommended response headers</Description>
 		<Title>Jhoose Security</Title>
@@ -81,6 +81,7 @@
             3.0.0 - Added multisite support, CSP and Permissions Policy, Security Headers can now be configured per site.
             3.0.4 - Fixed issue with the files not being copied to the output directory when building the project, this was causing the module to not work when installed from NuGet.
             3.1.0 - Added support for CMS13.
+            3.2.0 - Updated Purge scheduled job to run in batches : thanks @kennygutierrez
 		</ReleaseNotes>
 		<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
 		<RestoreSources Condition=" '$(Configuration)' == 'Debug' ">
@@ -93,7 +94,7 @@
       		https://nuget.pkg.github.com/andrewmarkham/index.json
 	  	</RestoreSources>
 		<Configurations>Debug;Release;PreRelease</Configurations>
-		<ReleaseVersion>3.1.0.0</ReleaseVersion>
+		<ReleaseVersion>3.2.0.0</ReleaseVersion>
         <NoWarn>1587, 1591</NoWarn>
 	</PropertyGroup>
 

src/Sample/CMS13/alloy13preview.csproj

@@ -13,10 +13,10 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="EPiServer.CMS" Version="13.0.0" />
-    <PackageReference Include="EPiServer.CMS.UI.AspNetIdentity" Version="13.0.0" />
-    <PackageReference Include="Optimizely.Graph.Cms" Version="13.0.0" />
-    <PackageReference Include="EPiServer.Cms.UI.ContentManager" Version="13.0.0" />
+    <PackageReference Include="EPiServer.CMS" Version="13.1.0" />
+    <PackageReference Include="EPiServer.CMS.UI.AspNetIdentity" Version="13.1.0" />
+    <PackageReference Include="Optimizely.Graph.Cms" Version="13.1.0" />
+    <PackageReference Include="EPiServer.Cms.UI.ContentManager" Version="13.1.0" />
 
     <PackageReference Include="Wangkanai.Detection" Version="8.7.0" />
   </ItemGroup>

src/Sample/CMS13/appsettings.json

@@ -12,9 +12,9 @@
     "ContentGraph": {
       "GatewayAddress": "https://cg.optimizely.com",
       "AllowSendingLog": "true",
-      "SingleKey": "CpnTa2chhSPeojBE3CVx1gyYIzfPWRkLYp4U3cxxAOUCjsfr",
-      "AppKey": "DgbIzCsKMBRSL4BbIfbcDBaLUPBra4IlxkJX1FtGNyDTIteU",
-      "Secret": "19ee4CmJyWlCjwS88fSkIVjrlKPaHFfoeANX86UWMyHu6QHw4eWhbPEB3m4sWXXL"
+      "SingleKey": "...",
+      "AppKey": "...",
+      "Secret": "..."
     }
   }
 }