Phase 5: workflow unification via PagedScrapeRunner (functional refactor plan)

src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs

@@ -49,6 +49,7 @@ public override void OnFrameworkInitializationCompleted()
 
                 return ctx.ScrapeConfiguration.GetScrapeConfigurations().ToAppModel();
             })
+            .AddSingleton(sp => sp.GetRequiredService<ScrapeConfiguration>().SearchConfiguration)
             .AddSingleton<LogBroadcaster>()
             .AddSingleton<ImageBroadcaster>()
             .AddSingleton(sp =>
@@ -91,6 +92,11 @@ public override void OnFrameworkInitializationCompleted()
             .AddSingleton<IPlaywrightService, PlaywrightService>()
             .AddTransient<SearchWorkflow>()
             .AddTransient<SearchResultsPage>()
+            .AddTransient<PagedScrapeRunner>()
+            .AddTransient<SubscriptionsImagesListPage>()
+            .AddTransient<SubscriptionsWorkflow>()
+            .AddTransient<ITopWallpapersPage, TopWallpapersPage>()
+            .AddTransient<TopWallpapersWorkflow>()
             .AddTransient<IImportExportService, ImportExportService>()
             .AddTransient<IFileSystem, RealFileSystem>()
             .AddTransient<ScrapeConfigurationService>()

src/AStar.Dev.Wallpaper.Scrapper/ScrapperConstants.cs

@@ -8,8 +8,6 @@ public static class ScrapperConstants
 
     public const float ThumbnailCornerRadius = 20f;
 
-    public static readonly TimeSpan PageNavigationDelay = TimeSpan.FromSeconds(2);
-
     public static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(10);
 
     public static readonly TimeSpan ImageAlreadyDownloadedDelay = TimeSpan.FromMilliseconds(500);

src/AStar.Dev.Wallpaper.Scrapper/Support/RandomDelayStrategy.cs

@@ -10,10 +10,13 @@ public Task DelayAsync(DelayKind delayKind, CancellationToken cancellationToken
         => delayKind switch
         {
             DelayKind.CategoryUpToDate => Task.Delay(TimeSpan.FromSeconds(new Random().Next(1, 5)), cancellationToken),
-            DelayKind.PageNavigation => Task.Delay(ScrapperConstants.PageNavigationDelay, cancellationToken),
+            DelayKind.PageNavigation => Task.Delay(TimeSpan.FromSeconds(RandomImagePauseSeconds()), cancellationToken),
             DelayKind.ImageAlreadyDownloaded => Task.Delay(ScrapperConstants.ImageAlreadyDownloadedDelay, cancellationToken),
-            DelayKind.BeforeImage => Task.Delay(TimeSpan.FromSeconds(Random.Shared.Next(scrapeConfiguration.SearchConfiguration.ImagePauseInSeconds, scrapeConfiguration.SearchConfiguration.ImagePauseInSeconds + 4)), cancellationToken),
+            DelayKind.BeforeImage => Task.Delay(TimeSpan.FromSeconds(RandomImagePauseSeconds()), cancellationToken),
             DelayKind.Retry => Task.Delay(ScrapperConstants.RetryDelay, cancellationToken),
             _ => throw new ArgumentOutOfRangeException(nameof(delayKind), delayKind, "Unrecognised delay kind."),
         };
+
+    private int RandomImagePauseSeconds()
+        => Random.Shared.Next(scrapeConfiguration.SearchConfiguration.ImagePauseInSeconds, scrapeConfiguration.SearchConfiguration.ImagePauseInSeconds + 4);
 }

src/AStar.Dev.Wallpaper.Scrapper/Workflows/PagedScrapePlan.cs

@@ -0,0 +1,19 @@
+using AStar.Dev.FunctionalParadigm;
+using AStar.Dev.Wallpaper.Scrapper.Models;
+
+namespace AStar.Dev.Wallpaper.Scrapper.Workflows;
+
+/// <summary>Describes one workflow's configuration of the shared <see cref="PagedScrapeRunner" /> page loop.</summary>
+/// <param name="StartingPage">The first page number to visit.</param>
+/// <param name="TotalPages">The last page number to visit, inclusive.</param>
+/// <param name="RecordProgress">Invoked with the current page number before the page is loaded, so the caller can record its own progress state.</param>
+/// <param name="LoadPageAsync">Loads the page identified by the supplied page number.</param>
+/// <param name="GetLinksAsync">Reads the image page links from the currently loaded page.</param>
+/// <param name="ProcessLinksAsync">Processes the image page links read from the currently loaded page.</param>
+public record PagedScrapePlan(
+    int StartingPage,
+    int TotalPages,
+    Action<int> RecordProgress,
+    Func<int, Task<Result<Unit, ScrapeError>>> LoadPageAsync,
+    Func<Task<Result<IReadOnlyCollection<string>, ScrapeError>>> GetLinksAsync,
+    Func<IReadOnlyCollection<string>, CancellationToken, Task<Result<Unit, ScrapeError>>> ProcessLinksAsync);

src/AStar.Dev.Wallpaper.Scrapper/Workflows/PagedScrapePlanFactory.cs

@@ -0,0 +1,26 @@
+using AStar.Dev.FunctionalParadigm;
+using AStar.Dev.Guard.Clauses;
+using AStar.Dev.Wallpaper.Scrapper.Models;
+
+namespace AStar.Dev.Wallpaper.Scrapper.Workflows;
+
+/// <summary>Factory methods for creating instances of <see cref="PagedScrapePlan" />.</summary>
+public static class PagedScrapePlanFactory
+{
+    /// <summary>Creates a <see cref="PagedScrapePlan" />.</summary>
+    public static PagedScrapePlan Create(
+        int startingPage,
+        int totalPages,
+        Action<int> recordProgress,
+        Func<int, Task<Result<Unit, ScrapeError>>> loadPageAsync,
+        Func<Task<Result<IReadOnlyCollection<string>, ScrapeError>>> getLinksAsync,
+        Func<IReadOnlyCollection<string>, CancellationToken, Task<Result<Unit, ScrapeError>>> processLinksAsync)
+    {
+        GuardAgainst.Null(recordProgress);
+        GuardAgainst.Null(loadPageAsync);
+        GuardAgainst.Null(getLinksAsync);
+        GuardAgainst.Null(processLinksAsync);
+
+        return new(startingPage, totalPages, recordProgress, loadPageAsync, getLinksAsync, processLinksAsync);
+    }
+}

src/AStar.Dev.Wallpaper.Scrapper/Workflows/PagedScrapeRunner.cs

@@ -0,0 +1,35 @@
+using AStar.Dev.FunctionalParadigm;
+using AStar.Dev.Guard.Clauses;
+using AStar.Dev.Wallpaper.Scrapper.Models;
+using AStar.Dev.Wallpaper.Scrapper.Support;
+
+namespace AStar.Dev.Wallpaper.Scrapper.Workflows;
+
+/// <summary>Runs the page loop shared by every paged scrape workflow: delay, record progress, save configuration, load the page, read its links, then process them.</summary>
+public sealed class PagedScrapeRunner(ConfigurationSaver configurationSaver, IDelayStrategy delayStrategy)
+{
+    /// <summary>Runs <paramref name="plan" /> from its starting page to its total page, inclusive, stopping at the first failing step.</summary>
+    public async Task<Result<Unit, ScrapeError>> RunAsync(PagedScrapePlan plan, CancellationToken ct = default)
+    {
+        GuardAgainst.Null(plan);
+
+        for (int pageNumber = plan.StartingPage; pageNumber <= plan.TotalPages; pageNumber++)
+        {
+            ct.ThrowIfCancellationRequested();
+            await delayStrategy.DelayAsync(DelayKind.PageNavigation, ct).ConfigureAwait(false);
+            plan.RecordProgress(pageNumber);
+
+            var pageResult = await configurationSaver.SaveUpdatedConfigurationAsync()
+                .BindAsync(_ => plan.LoadPageAsync(pageNumber))
+                .BindAsync(_ => plan.GetLinksAsync())
+                .BindAsync(links => plan.ProcessLinksAsync(links, ct))
+                .ConfigureAwait(false);
+
+            var pageFailed = pageResult.Match(_ => false, _ => true);
+
+            if (pageFailed) return pageResult;
+        }
+
+        return Unit.Value;
+    }
+}

src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchWorkflow.cs

@@ -8,7 +8,7 @@
 
 namespace AStar.Dev.Wallpaper.Scrapper.Workflows;
 
-public sealed class SearchWorkflow(SearchResultsPage searchResultsPage, ScrapeConfiguration injectedScrapeConfiguration, ConfigurationSaver configurationSaver, ImagePageService imagePageService, IDirectoryHelper directoryHelper, ILogger logger, IDelayStrategy delayStrategy, TimeProvider timeProvider)
+public sealed class SearchWorkflow(SearchResultsPage searchResultsPage, ScrapeConfiguration injectedScrapeConfiguration, ConfigurationSaver configurationSaver, ImagePageService imagePageService, IDirectoryHelper directoryHelper, ILogger logger, IDelayStrategy delayStrategy, TimeProvider timeProvider, PagedScrapeRunner pagedScrapeRunner)
 {
     private SearchProgress progress = null!;
 
@@ -84,26 +84,22 @@ private async Task<Result<Unit, ScrapeError>> ProcessAllCategoryPagesAsync(Categ
         long startTimestamp = timeProvider.GetTimestamp();
         logger.Debug("About to visit the specific {Category} pages now...", searchCategory.Name);
 
-        for (int currentPageNumber = progress.SearchConfiguration.StartingPageNumber; currentPageNumber <= progress.SearchConfiguration.TotalPages; currentPageNumber++)
-        {
-            await delayStrategy.DelayAsync(DelayKind.PageNavigation, ct).ConfigureAwait(false);
-            logger.Debug("About to visit page {page} (of {totalPages}) for {Category} now...", currentPageNumber, progress.SearchConfiguration.TotalPages, searchCategory.Name);
-            progress = progress with { SearchConfiguration = progress.SearchConfiguration with { StartingPageNumber = currentPageNumber, }, };
-            searchCategory.LastPageVisited = currentPageNumber;
-
-            var pageResult = await configurationSaver.SaveUpdatedConfigurationAsync()
-                .BindAsync(_ => searchResultsPage.LoadSearchPageAsync(combinedSearchString, currentPageNumber))
-                .BindAsync(_ => searchResultsPage.ImagePageLinksAsync())
-                .BindAsync(links => imagePageService.GetTheImagePagesAsync(links, searchCategory.Id, searchCategory.Name, ct))
-                .ConfigureAwait(false);
-
-            var pageFailed = pageResult.Match(_ => false, _ => true);
-
-            if (pageFailed) return pageResult;
-        }
+        var plan = PagedScrapePlanFactory.Create(
+            progress.SearchConfiguration.StartingPageNumber,
+            progress.SearchConfiguration.TotalPages,
+            pageNumber => RecordCategoryPageProgress(searchCategory, pageNumber),
+            pageNumber => searchResultsPage.LoadSearchPageAsync(combinedSearchString, pageNumber),
+            () => searchResultsPage.ImagePageLinksAsync(),
+            (links, innerCt) => imagePageService.GetTheImagePagesAsync(links, searchCategory.Id, searchCategory.Name, innerCt));
 
-        logger.Information("Completed visiting the {Category}. Total time: {CategoryVisitDuration}", searchCategory.Name, timeProvider.GetElapsedTime(startTimestamp));
+        return (await pagedScrapeRunner.RunAsync(plan, ct).ConfigureAwait(false))
+            .Tap(_ => logger.Information("Completed visiting the {Category}. Total time: {CategoryVisitDuration}", searchCategory.Name, timeProvider.GetElapsedTime(startTimestamp)));
+    }
 
-        return Unit.Value;
+    private void RecordCategoryPageProgress(Category searchCategory, int pageNumber)
+    {
+        logger.Debug("About to visit page {page} (of {totalPages}) for {Category} now...", pageNumber, progress.SearchConfiguration.TotalPages, searchCategory.Name);
+        progress = progress with { SearchConfiguration = progress.SearchConfiguration with { StartingPageNumber = pageNumber, }, };
+        searchCategory.LastPageVisited = pageNumber;
     }
 }

src/AStar.Dev.Wallpaper.Scrapper/Workflows/SubscriptionsWorkflow.cs

@@ -3,79 +3,81 @@
 using AStar.Dev.Wallpaper.Scrapper.Pages;
 using AStar.Dev.Wallpaper.Scrapper.Services;
 using AStar.Dev.Wallpaper.Scrapper.Support;
-using Serilog.Core;
+using Serilog;
 
 namespace AStar.Dev.Wallpaper.Scrapper.Workflows;
 
 public sealed class SubscriptionsWorkflow(
     SubscriptionsImagesListPage subscriptionsImagesListPage,
     ImagePageService imagePageService,
-    SearchConfiguration searchConfiguration,
-    ScrapeDirectories scrapeDirectories,
+    ScrapeConfiguration scrapeConfiguration,
     ConfigurationSaver configurationSaver,
-    Logger logger)
+    PagedScrapeRunner pagedScrapeRunner,
+    ILogger logger)
 {
-    private SearchConfiguration _searchConfiguration = searchConfiguration;
-    private ScrapeDirectories _scrapeDirectories = scrapeDirectories;
+    private const int FirstPageNumber = 1;
 
-    public async Task RunAsync(CancellationToken ct = default)
+    private SearchConfiguration searchConfiguration = scrapeConfiguration.SearchConfiguration;
+    private ScrapeDirectories scrapeDirectories = scrapeConfiguration.ScrapeDirectories;
+
+    public Task<Result<Unit, ScrapeError>> RunAsync(CancellationToken ct = default)
+        => RunSubscriptionsAsync(ct).LogFailure(logger);
+
+    private async Task<Result<Unit, ScrapeError>> RunSubscriptionsAsync(CancellationToken ct)
     {
-        try
-        {
-            await GetTheNewSubscriptionImagesAsync(ct).ConfigureAwait(false);
-        }
-        catch (Exception exception) when (exception is not OperationCanceledException)
-        {
-            logger.Error(exception.GetBaseException().Message);
-            throw;
-        }
+        searchConfiguration = searchConfiguration with { SubscriptionsStartingPageNumber = FirstPageNumber, };
+
+        await LoadStartingPageAsync().ConfigureAwait(false);
+
+        return await subscriptionsImagesListPage.PageInfoAsync()
+            .BindAsync(pageInfo => ProcessSubscriptionsAsync(pageInfo, ct))
+            .ConfigureAwait(false);
     }
 
-    private async Task GetTheNewSubscriptionImagesAsync(CancellationToken ct)
+    private async Task LoadStartingPageAsync()
     {
-        _searchConfiguration = _searchConfiguration with { SubscriptionsStartingPageNumber = 1 };
-        var pageDetails = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(_searchConfiguration.SubscriptionsStartingPageNumber).ConfigureAwait(false);
-        var loadedSuccessfully = pageDetails.Match(_ => true, _ => false);
+        var loadResult = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(FirstPageNumber).ConfigureAwait(false);
+        var loadedSuccessfully = loadResult.Match(_ => true, _ => false);
 
-        if (!loadedSuccessfully) _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(1).ConfigureAwait(false);
-
-        var pageInfo = (await subscriptionsImagesListPage.PageInfoAsync().ConfigureAwait(false))
-            .Match(info => info, error => throw new InvalidOperationException(error.Message));
+        if (!loadedSuccessfully) _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(FirstPageNumber).ConfigureAwait(false);
+    }
 
-        if (pageInfo.SubDirectoryName.Length > 0) _scrapeDirectories = _scrapeDirectories with { SubDirectoryName = pageInfo.SubDirectoryName };
+    private async Task<Result<Unit, ScrapeError>> ProcessSubscriptionsAsync(PageInfo pageInfo, CancellationToken ct)
+    {
+        if (pageInfo.SubDirectoryName.Length > 0) scrapeDirectories = scrapeDirectories with { SubDirectoryName = pageInfo.SubDirectoryName, };
 
-        UpdateSearchTotalPagesIfRequired(pageInfo.PageCount);
+        if (searchConfiguration.SubscriptionsTotalPages != pageInfo.PageCount) searchConfiguration = searchConfiguration with { SubscriptionsTotalPages = pageInfo.PageCount, };
 
         await configurationSaver.SaveUpdatedConfigurationAsync().ConfigureAwait(false);
 
-        for (int currentPageNumber = _searchConfiguration.SubscriptionsStartingPageNumber;
-            currentPageNumber <= _searchConfiguration.SubscriptionsTotalPages;
-            currentPageNumber++)
-        {
-            ct.ThrowIfCancellationRequested();
-            int delay = Random.Shared.Next(_searchConfiguration.ImagePauseInSeconds, _searchConfiguration.ImagePauseInSeconds + 4);
-            await Task.Delay(TimeSpan.FromSeconds(delay), ct).ConfigureAwait(false);
-            _searchConfiguration = _searchConfiguration with { SubscriptionsStartingPageNumber = currentPageNumber };
-            await configurationSaver.SaveUpdatedConfigurationAsync().ConfigureAwait(false);
-            logger.Information("Getting page {subscriptionPage} (of {totalPagesForSubscriptions}) now.", currentPageNumber, _searchConfiguration.SubscriptionsTotalPages);
-            _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(currentPageNumber).ConfigureAwait(false);
-
-            var imagePageLinks = (await subscriptionsImagesListPage.GetImagePageLinksAsync().ConfigureAwait(false))
-                .Match(links => links, error => throw new InvalidOperationException(error.Message));
-
-            _ = (await imagePageService.GetTheImagePagesAsync(imagePageLinks, "", pageInfo.SubDirectoryName, ct: ct).ConfigureAwait(false))
-                .Match(unit => unit, error => throw new InvalidOperationException(error.Message));
-        }
-
-        if (pageInfo.PageCount > 0)
-        {
-            _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(1).ConfigureAwait(false);
-            _ = await subscriptionsImagesListPage.ClearAsync().ConfigureAwait(false);
-        }
+        var plan = PagedScrapePlanFactory.Create(
+            searchConfiguration.SubscriptionsStartingPageNumber,
+            searchConfiguration.SubscriptionsTotalPages,
+            _ => { },
+            pageNumber => LoadSubscriptionsPageAsync(pageNumber, searchConfiguration.SubscriptionsTotalPages),
+            subscriptionsImagesListPage.GetImagePageLinksAsync,
+            (links, innerCt) => imagePageService.GetTheImagePagesAsync(links, string.Empty, pageInfo.SubDirectoryName, innerCt));
+
+        return await pagedScrapeRunner.RunAsync(plan, ct)
+            .BindAsync(_ => ClearSubscriptionsIfCompleteAsync(pageInfo))
+            .ConfigureAwait(false);
     }
 
-    private void UpdateSearchTotalPagesIfRequired(int pageCount)
+    private async Task<Result<Unit, ScrapeError>> LoadSubscriptionsPageAsync(int pageNumber, int totalPages)
     {
-        if (_searchConfiguration.SubscriptionsTotalPages != pageCount) _searchConfiguration = _searchConfiguration with { SubscriptionsTotalPages = pageCount };
+        logger.Information("Getting page {subscriptionPage} (of {totalPagesForSubscriptions}) now.", pageNumber, totalPages);
+        _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(pageNumber).ConfigureAwait(false);
+
+        return Unit.Value;
+    }
+
+    private async Task<Result<Unit, ScrapeError>> ClearSubscriptionsIfCompleteAsync(PageInfo pageInfo)
+    {
+        if (pageInfo.PageCount <= 0) return Unit.Value;
+
+        _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(FirstPageNumber).ConfigureAwait(false);
+        _ = await subscriptionsImagesListPage.ClearAsync().ConfigureAwait(false);
+
+        return Unit.Value;
     }
 }