Phase 3: extract pure cores + inject functions (functional refactor plan)

docs/wallpaper-scrapper-functional-refactor-plan.md

@@ -151,7 +151,7 @@ Plus `ScrapeErrorFactory` per DU convention, and one logging extension `Result<T
 
 Every phase follows the repo's non-negotiable TDD workflow: **c-sharp-qa writes failing tests → commit → c-sharp-dev implements → all tests green → c-sharp-reviewer reviews → new c-sharp-dev fixes findings**. Tests come first in every phase — there is no after-the-fact test pass. Standing test rules: never against the real database (in-memory SQLite via `IDbContextFactory`), NSubstitute for collaborators, zero-delay strategy injected into workflow tests, Shouldly assertions, `Given…`/`when_…_then_…` naming. One commit (or PR) per phase; `graphify update .` after each phase.
 
-### Phase 0 — Bug fixes (TDD, before any refactor)
+### Phase 0 — Bug fixes (TDD, before any refactor) - complete
 
 Fix B1–B5 from §3.3, each with a failing test first:
 
@@ -162,11 +162,11 @@ Fix B1–B5 from §3.3, each with a failing test first:
 - **B5:** test that when control-reset fails the workflow is **not** invoked. Fix ahead of the full Phase 6 rework: failure lambda returns a `Fail` result.
 - **B6:** DB connection string becomes configurable (`ConnectionStrings:Sqlite` from configuration/user secrets) with `/home/jbarden/Documents/Scrapper/scrapper.db` as the default. Tests: DI-composed connection string uses the configured value when present, the new default otherwise. Fresh database — no data migration; old data re-imported manually via the existing Import buttons after first run.
 
-### Phase 1 — FunctionalParadigm additions
+### Phase 1 — FunctionalParadigm additions - complete
 
 Implement §4.1–4.4 with full test coverage (Match/Map/Bind laws, error accumulation, cancellation passthrough, async overloads). No consumer changes yet — additive, zero risk.
 
-### Phase 2 — Deletions, merges, renames
+### Phase 2 — Deletions, merges, renames - complete
 
 1. Delete: `SearchWorkflow` (old), `ConfigurationSaverFunctional`, `ImagePageServiceFunctional` (+ its interface), `ImagePageResultFunctional` (+ interface), `TopWallpapersWorkflowFunctional`, `SearchResultsPage` (old twin).
 2. Merge `TopWallpapersPageFunctional` into `TopWallpapersPage` (keeping the `page ??=` init fix and the `ITopWallpapersPage` interface, renamed from `ITopWallpapersPageFunctional`).

src/AStar.Dev.Wallpaper.Scrapper/AStar.Dev.Wallpaper.Scrapper.csproj

@@ -23,6 +23,7 @@
         <PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
         <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
         <PackageReference Include="Microsoft.Extensions.Hosting" />
+        <PackageReference Include="Microsoft.Extensions.Http" />
         <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
         <PackageReference Include="Microsoft.Playwright" />
         <PackageReference Include="Serilog.Exceptions" />

src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs

@@ -97,6 +97,10 @@ public override void OnFrameworkInitializationCompleted()
             .AddTransient<ImagePageService>()
             .AddTransient<ImagePage>()
             .AddSingleton<IDirectoryHelper, DirectoryHelper>()
+            .AddSingleton<IDelayStrategy, RandomDelayStrategy>()
+            .AddTransient<IImageSaver, ImageSaver>()
+            .AddHttpClient<IImageRetriever, ImageRetriever>(client => client.Timeout = TimeSpan.FromMinutes(2))
+            .Services
             .AddTransient(_ => TimeProvider.System)
             .AddTransient<Func<ScrapeConfigurationView>>(sp => () => sp.GetRequiredService<ScrapeConfigurationView>())
             .AddTransient<Func<ClassificationsView>>(sp => () => sp.GetRequiredService<ClassificationsView>())

src/AStar.Dev.Wallpaper.Scrapper/Models/PageInfo.cs

@@ -0,0 +1,7 @@
+namespace AStar.Dev.Wallpaper.Scrapper.Models;
+
+/// <summary>The information parsed from a search-results-style page header.</summary>
+/// <param name="PageCount">The total number of pages available.</param>
+/// <param name="ImageCount">The total number of images reported by the header.</param>
+/// <param name="SubDirectoryName">The sub-directory name derived from the header text.</param>
+public record PageInfo(int PageCount, int ImageCount, string SubDirectoryName);

src/AStar.Dev.Wallpaper.Scrapper/Models/PageInfoFactory.cs

@@ -0,0 +1,15 @@
+using AStar.Dev.Guard.Clauses;
+
+namespace AStar.Dev.Wallpaper.Scrapper.Models;
+
+/// <summary>Factory methods for creating instances of <see cref="PageInfo" />.</summary>
+public static class PageInfoFactory
+{
+    /// <summary>Creates a <see cref="PageInfo" />.</summary>
+    public static PageInfo Create(int pageCount, int imageCount, string subDirectoryName)
+    {
+        GuardAgainst.Null(subDirectoryName);
+
+        return new(pageCount, imageCount, subDirectoryName);
+    }
+}

src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeError.cs

@@ -0,0 +1,38 @@
+namespace AStar.Dev.Wallpaper.Scrapper.Models;
+
+/// <summary>Base type for every error that a scrape pipeline operation can fail with.</summary>
+/// <param name="Message">A human-readable description of the failure.</param>
+public abstract record ScrapeError(string Message);
+
+/// <summary>The requested page could not be loaded.</summary>
+/// <param name="Url">The URL that failed to load.</param>
+/// <param name="Message">A human-readable description of the failure.</param>
+public sealed record PageLoadFailed(string Url, string Message) : ScrapeError(Message);
+
+/// <summary>The page header text could not be parsed into a <see cref="PageInfo" />.</summary>
+/// <param name="HeaderText">The header text that failed to parse, when available.</param>
+/// <param name="Message">A human-readable description of the failure.</param>
+public sealed record PageParseFailed(string? HeaderText, string Message) : ScrapeError(Message);
+
+/// <summary>The image could not be downloaded.</summary>
+/// <param name="ImageUrl">The URL of the image that failed to download.</param>
+/// <param name="Message">A human-readable description of the failure.</param>
+public sealed record ImageDownloadFailed(string ImageUrl, string Message) : ScrapeError(Message);
+
+/// <summary>The downloaded image could not be saved to disk.</summary>
+/// <param name="Path">The path the image was being saved to.</param>
+/// <param name="Message">A human-readable description of the failure.</param>
+public sealed record ImageSaveFailed(string Path, string Message) : ScrapeError(Message);
+
+/// <summary>The updated scrape configuration could not be saved.</summary>
+/// <param name="Message">A human-readable description of the failure.</param>
+public sealed record ConfigurationSaveFailed(string Message) : ScrapeError(Message);
+
+/// <summary>The file could not be classified.</summary>
+/// <param name="FileName">The name of the file that failed to classify.</param>
+/// <param name="Message">A human-readable description of the failure.</param>
+public sealed record ClassificationFailed(string FileName, string Message) : ScrapeError(Message);
+
+/// <summary>An unanticipated exception was raised while running the scrape pipeline.</summary>
+/// <param name="Exception">The exception that was raised.</param>
+public sealed record UnexpectedError(Exception Exception) : ScrapeError(Exception.Message);

src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorFactory.cs

@@ -0,0 +1,67 @@
+using AStar.Dev.Guard.Clauses;
+
+namespace AStar.Dev.Wallpaper.Scrapper.Models;
+
+/// <summary>Factory methods for creating instances of the <see cref="ScrapeError" /> discriminated union.</summary>
+public static class ScrapeErrorFactory
+{
+    /// <summary>Creates a <see cref="PageLoadFailed" /> error.</summary>
+    public static PageLoadFailed CreatePageLoadFailed(string url, string message)
+    {
+        GuardAgainst.Null(url);
+        GuardAgainst.Null(message);
+
+        return new(url, message);
+    }
+
+    /// <summary>Creates a <see cref="PageParseFailed" /> error.</summary>
+    public static PageParseFailed CreatePageParseFailed(string? headerText, string message)
+    {
+        GuardAgainst.Null(message);
+
+        return new(headerText, message);
+    }
+
+    /// <summary>Creates an <see cref="ImageDownloadFailed" /> error.</summary>
+    public static ImageDownloadFailed CreateImageDownloadFailed(string imageUrl, string message)
+    {
+        GuardAgainst.Null(imageUrl);
+        GuardAgainst.Null(message);
+
+        return new(imageUrl, message);
+    }
+
+    /// <summary>Creates an <see cref="ImageSaveFailed" /> error.</summary>
+    public static ImageSaveFailed CreateImageSaveFailed(string path, string message)
+    {
+        GuardAgainst.Null(path);
+        GuardAgainst.Null(message);
+
+        return new(path, message);
+    }
+
+    /// <summary>Creates a <see cref="ConfigurationSaveFailed" /> error.</summary>
+    public static ConfigurationSaveFailed CreateConfigurationSaveFailed(string message)
+    {
+        GuardAgainst.Null(message);
+
+        return new(message);
+    }
+
+    /// <summary>Creates a <see cref="ClassificationFailed" /> error.</summary>
+    public static ClassificationFailed CreateClassificationFailed(string fileName, string message)
+    {
+        GuardAgainst.Null(fileName);
+        GuardAgainst.Null(message);
+
+        return new(fileName, message);
+    }
+
+    /// <summary>Creates an <see cref="UnexpectedError" /> error.</summary>
+    public static UnexpectedError CreateUnexpectedError(Exception exception)
+    {
+        GuardAgainst.Null(exception);
+
+        return new(exception);
+    }
+}

src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorLoggingExtensions.cs

@@ -0,0 +1,12 @@
+using AStar.Dev.FunctionalParadigm;
+using Serilog;
+
+namespace AStar.Dev.Wallpaper.Scrapper.Models;
+
+/// <summary>Logging extensions for <see cref="Result{TResult,TError}" /> pipelines that fail with a <see cref="ScrapeError" />.</summary>
+public static class ScrapeErrorLoggingExtensions
+{
+    /// <summary>Logs an error when <paramref name="result" /> is a failure, then returns <paramref name="result" /> unchanged.</summary>
+    public static Result<TResult, ScrapeError> LogFailure<TResult>(this Result<TResult, ScrapeError> result, ILogger logger)
+        => result.Tap(_ => { }, error => logger.Error("Scrape pipeline failure: {Message}", error.Message));
+}

src/AStar.Dev.Wallpaper.Scrapper/Pages/ImageLinkSelector.cs

@@ -0,0 +1,11 @@
+namespace AStar.Dev.Wallpaper.Scrapper.Pages;
+
+/// <summary>Selects the wanted image links from a page's raw anchor <c>href</c> attributes.</summary>
+public static class ImageLinkSelector
+{
+    /// <summary>Selects the non-null <paramref name="hrefs" /> that point at an image, up to <see cref="ScrapperConstants.ImagesPerPage" />.</summary>
+    public static IReadOnlyCollection<string> SelectWanted(IEnumerable<string?> hrefs)
+        => [.. hrefs.Where(href => href is not null && href.Contains("/w/", StringComparison.Ordinal))
+                    .Select(href => href!)
+                    .Take(ScrapperConstants.ImagesPerPage)];
+}

src/AStar.Dev.Wallpaper.Scrapper/Pages/ImagePage.cs

@@ -3,6 +3,7 @@
 using AStar.Dev.Wallpaper.Scrapper.Models;
 using AStar.Dev.Wallpaper.Scrapper.Repositories;
 using AStar.Dev.Wallpaper.Scrapper.Services;
+using AStar.Dev.Wallpaper.Scrapper.Support;
 using Microsoft.Playwright;
 
 namespace AStar.Dev.Wallpaper.Scrapper.Pages;
@@ -17,120 +18,37 @@ public async Task<ImagePageResult> GetImageFromPageAsync(string link, string cat
         _ = await page.GotoAsync(link);
 
         var tagLocators = await page.Locator(".tagname").AllAsync();
-        string directoryName = scrapeConfiguration.ScrapeDirectories.BaseSaveDirectory.CombinePath(categoryName.Replace(' ', '-'));
-        var (directoryNameUpdated, filePrefix, skip, imageTags) = await ProcessTheImageTagsAsync(tagLocators, [directoryName]);
-
-        if (skip) return new ImagePageResult(null, directoryNameUpdated, filePrefix, skip, imageTags);
-
-        var imageTag = page.Locator("#wallpaper");
-        string? sourcePath = await imageTag.GetAttributeAsync("src");
-
-        return new ImagePageResult(sourcePath, directoryNameUpdated, filePrefix, skip, imageTags);
-    }
-
-    private async Task<(List<string> directoryName, string filePrefix, bool skip, IReadOnlyList<string> tags)> ProcessTheImageTagsAsync(IEnumerable<ILocator> tags, List<string> directoryName)
-    {
-        bool skip = false;
-        string filePrefix = string.Empty;
-
-        var tagData = await Task.WhenAll(tags.Select(GetTagsAsync));
-        var imageTags = tagData.Select(t => t.Tag).Where(t => !string.IsNullOrWhiteSpace(t)).ToList();
+        var tagData = await Task.WhenAll(tagLocators.Select(GetTagsAsync));
 
         await scrapedTagRepository.SaveAsync([.. tagData.Where(t => !string.IsNullOrWhiteSpace(t.Category))]);
 
-        foreach (var (tagText, tagToUse) in tagData)
-        {
-            if (tagToUse == null) continue;
-
-            string trimmedTagToUse = tagToUse.Trim();
-            skip = IsOneOfTheImageTagsToExcludeCompletely(trimmedTagToUse) || IsOneOfTheImageTagsToExcludeCompletely(tagText);
+        string initialDirectory = scrapeConfiguration.ScrapeDirectories.BaseSaveDirectory.CombinePath(categoryName.Replace(' ', '-'));
+        var context = TagRuleContextFactory.Create(initialDirectory, scrapeConfiguration.ScrapeDirectories.BaseDirectoryFamous, tagsToIgnoreCompletely, tagsTextToIgnore);
 
-            if (skip) break;
+        var outcome = TagRules.Evaluate(tagData, context);
 
-            var (filePrefixUpdated, directoryNameUpdated) = UpdateFilePrefixForModels(trimmedTagToUse, tagText, filePrefix, directoryName);
-
-            directoryName = directoryNameUpdated;
-
-            filePrefix = UpdateFilePrefixForVehicles(trimmedTagToUse, filePrefixUpdated);
-
-            if (UpdateToTagIsNotRequired(trimmedTagToUse, tagText, filePrefix)) continue;
-
-            filePrefix = string.Join("-", filePrefix, tagText.Replace(' ', '-')).ToLowerInvariant();
-            directoryName.Add(scrapeConfiguration.ScrapeDirectories.BaseDirectoryFamous);
-        }
-
-        filePrefix = UpdateFilePrefixIfRequired(filePrefix);
-
-        return (directoryName, filePrefix, skip, imageTags);
+        return await MapOutcomeToResultAsync(outcome);
     }
 
-    private static string UpdateFilePrefixIfRequired(string filePrefix)
+    private async Task<ImagePageResult> MapOutcomeToResultAsync(TagOutcome outcome)
+        => outcome switch
+        {
+            SkipImage skip => new ImagePageResult(null, [], string.Empty, true, skip.Tags),
+            Accept accept => new ImagePageResult(await GetImageSourceAsync(), [.. accept.DirectorySegments], accept.FilePrefix, false, accept.Tags),
+            _ => throw new InvalidOperationException("Unexpected tag outcome."),
+        };
+
+    private async Task<string?> GetImageSourceAsync()
     {
-        if (filePrefix.StartsWith('-')) filePrefix = filePrefix[1..];
+        var imageTag = page.Locator("#wallpaper");
 
-        return filePrefix;
+        return await imageTag.GetAttributeAsync("src");
     }
 
-    private bool UpdateToTagIsNotRequired(string tagToUse, string tagText, string filePrefix)
-        => TagIsNotCelebEtc(tagToUse) || FilePrefixDoesNotNeedUpdating(tagText, filePrefix);
-
     private static async Task<TagData> GetTagsAsync(ILocator tag)
     {
         string textTask = await tag.InnerTextAsync();
         string? attrTask = await tag.GetAttributeAsync("original-title");
         return new TagData(textTask, attrTask);
     }
-
-    private bool FilePrefixDoesNotNeedUpdating(string tagText, string filePrefix)
-        => IsWantedText(tagText) || !filePrefix.Contains(tagText);
-
-    private static bool TagIsNotCelebEtc(string tagToUse)
-        => !TagContains(tagToUse, "celeb")
-           && !TagContains(tagToUse, "singer")
-           && !TagContains(tagToUse, "actress");
-
-    private static bool TagContains(string tagToUse, string contains)
-        => tagToUse.Contains(contains, StringComparison.OrdinalIgnoreCase);
-
-    private string UpdateFilePrefixForVehicles(string tagToUse, string filePrefix)
-    {
-        if (!TagContains(tagToUse, "Vehicles > Cars & Motorcycles")) return filePrefix;
-
-        if (IsWantedFilePrefix(tagToUse, filePrefix)) filePrefix = string.Join("-", filePrefix, tagToUse);
-
-        return filePrefix;
-    }
-
-    private bool IsWantedFilePrefix(string tagToUse, string filePrefix)
-        => IsWantedText(tagToUse) && !filePrefix.Contains(tagToUse) &&
-           !tagToUse.Equals("car", StringComparison.OrdinalIgnoreCase) &&
-           !TagContains(tagToUse, "cars");
-
-    private (string filePrefix, List<string> directoryName) UpdateFilePrefixForModels(string tagToUse, string tagText, string filePrefix, List<string> directoryName)
-    {
-        string filePrefixUpdated = filePrefix;
-
-        if (IsPeopleTag(tagToUse))
-        {
-            if (IsWantedText(tagText) && !filePrefix.Contains(tagText))
-            {
-
-                if (!directoryName.Contains(tagText) && !filePrefix.Contains(tagText, StringComparison.OrdinalIgnoreCase))
-                {
-                    filePrefixUpdated = string.Join("-", filePrefix, tagText);
-                    //directoryName.Add(tagText);
-                }
-            }
-        }
-
-        return (filePrefixUpdated, directoryName);
-    }
-
-    private static bool IsPeopleTag(string tagToUse) => TagContains(tagToUse, "people > model") || TagContains(tagToUse, "people > porn") || TagContains(tagToUse, "people > actress") || TagContains(tagToUse, "people > actor") || TagContains(tagToUse, "people > singer");
-
-    private bool IsOneOfTheImageTagsToExcludeCompletely(string tagText)
-        => tagsToIgnoreCompletely.Tags.Contains(tagText);
-
-    private bool IsWantedText(string tagText)
-        => !tagsTextToIgnore.Tags.Contains(tagText) && !tagText.StartsWith("model", StringComparison.OrdinalIgnoreCase);
 }

src/AStar.Dev.Wallpaper.Scrapper/Pages/PageHeaderParser.cs

@@ -0,0 +1,35 @@
+using System.Globalization;
+using AStar.Dev.FunctionalParadigm;
+using AStar.Dev.Wallpaper.Scrapper.Models;
+
+namespace AStar.Dev.Wallpaper.Scrapper.Pages;
+
+/// <summary>Parses the header text shown on a search-results page into a <see cref="PageInfo" />.</summary>
+public static class PageHeaderParser
+{
+    /// <summary>Parses <paramref name="headerText" />, frozen from the historical <c>SearchResultsPage.GetPageInfoAsync</c> parsing quirks.</summary>
+    public static Result<PageInfo, ScrapeError> Parse(string? headerText)
+    {
+        if (string.IsNullOrEmpty(headerText))
+            return ScrapeErrorFactory.CreatePageParseFailed(headerText, "The search results page header text was missing.");
+
+        try
+        {
+            var firstSpaceIndex = headerText.IndexOf(' ');
+            var hashIndex = headerText.IndexOf("for ", StringComparison.Ordinal) + 3;
+            var subDirectoryName = string.Empty;
+
+            if (hashIndex > 0) subDirectoryName = headerText[(hashIndex + 1)..].Replace(" ", "-").Replace("#", string.Empty);
+
+            var searchResults = headerText.Replace(",", string.Empty)[..firstSpaceIndex];
+            var imageCount = decimal.Parse(searchResults, CultureInfo.InvariantCulture);
+            var pageCount = Convert.ToInt32(Math.Ceiling(imageCount / ScrapperConstants.ImagesPerPage));
+
+            return PageInfoFactory.Create(pageCount, (int)imageCount, subDirectoryName);
+        }
+        catch (Exception exception)
+        {
+            return ScrapeErrorFactory.CreatePageParseFailed(headerText, exception.Message);
+        }
+    }
+}