diff --git a/docs/wallpaper-scrapper-functional-refactor-plan.md b/docs/wallpaper-scrapper-functional-refactor-plan.md index 50323d9..ec48b30 100644 --- a/docs/wallpaper-scrapper-functional-refactor-plan.md +++ b/docs/wallpaper-scrapper-functional-refactor-plan.md @@ -151,7 +151,7 @@ Plus `ScrapeErrorFactory` per DU convention, and one logging extension `Result + diff --git a/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs b/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs index fe09057..30dfbed 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs @@ -97,6 +97,10 @@ public override void OnFrameworkInitializationCompleted() .AddTransient() .AddTransient() .AddSingleton() + .AddSingleton() + .AddTransient() + .AddHttpClient(client => client.Timeout = TimeSpan.FromMinutes(2)) + .Services .AddTransient(_ => TimeProvider.System) .AddTransient>(sp => () => sp.GetRequiredService()) .AddTransient>(sp => () => sp.GetRequiredService()) diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Models/PageInfo.cs b/src/AStar.Dev.Wallpaper.Scrapper/Models/PageInfo.cs new file mode 100644 index 0000000..449a0bd --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Models/PageInfo.cs @@ -0,0 +1,7 @@ +namespace AStar.Dev.Wallpaper.Scrapper.Models; + +/// The information parsed from a search-results-style page header. +/// The total number of pages available. +/// The total number of images reported by the header. +/// The sub-directory name derived from the header text. +public record PageInfo(int PageCount, int ImageCount, string SubDirectoryName); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Models/PageInfoFactory.cs b/src/AStar.Dev.Wallpaper.Scrapper/Models/PageInfoFactory.cs new file mode 100644 index 0000000..9fb1606 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Models/PageInfoFactory.cs @@ -0,0 +1,15 @@ +using AStar.Dev.Guard.Clauses; + +namespace AStar.Dev.Wallpaper.Scrapper.Models; + +/// Factory methods for creating instances of . +public static class PageInfoFactory +{ + /// Creates a . + public static PageInfo Create(int pageCount, int imageCount, string subDirectoryName) + { + GuardAgainst.Null(subDirectoryName); + + return new(pageCount, imageCount, subDirectoryName); + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeError.cs b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeError.cs new file mode 100644 index 0000000..1fe4332 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeError.cs @@ -0,0 +1,38 @@ +namespace AStar.Dev.Wallpaper.Scrapper.Models; + +/// Base type for every error that a scrape pipeline operation can fail with. +/// A human-readable description of the failure. +public abstract record ScrapeError(string Message); + +/// The requested page could not be loaded. +/// The URL that failed to load. +/// A human-readable description of the failure. +public sealed record PageLoadFailed(string Url, string Message) : ScrapeError(Message); + +/// The page header text could not be parsed into a . +/// The header text that failed to parse, when available. +/// A human-readable description of the failure. +public sealed record PageParseFailed(string? HeaderText, string Message) : ScrapeError(Message); + +/// The image could not be downloaded. +/// The URL of the image that failed to download. +/// A human-readable description of the failure. +public sealed record ImageDownloadFailed(string ImageUrl, string Message) : ScrapeError(Message); + +/// The downloaded image could not be saved to disk. +/// The path the image was being saved to. +/// A human-readable description of the failure. +public sealed record ImageSaveFailed(string Path, string Message) : ScrapeError(Message); + +/// The updated scrape configuration could not be saved. +/// A human-readable description of the failure. +public sealed record ConfigurationSaveFailed(string Message) : ScrapeError(Message); + +/// The file could not be classified. +/// The name of the file that failed to classify. +/// A human-readable description of the failure. +public sealed record ClassificationFailed(string FileName, string Message) : ScrapeError(Message); + +/// An unanticipated exception was raised while running the scrape pipeline. +/// The exception that was raised. +public sealed record UnexpectedError(Exception Exception) : ScrapeError(Exception.Message); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorFactory.cs b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorFactory.cs new file mode 100644 index 0000000..8191d23 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorFactory.cs @@ -0,0 +1,67 @@ +using AStar.Dev.Guard.Clauses; + +namespace AStar.Dev.Wallpaper.Scrapper.Models; + +/// Factory methods for creating instances of the discriminated union. +public static class ScrapeErrorFactory +{ + /// Creates a error. + public static PageLoadFailed CreatePageLoadFailed(string url, string message) + { + GuardAgainst.Null(url); + GuardAgainst.Null(message); + + return new(url, message); + } + + /// Creates a error. + public static PageParseFailed CreatePageParseFailed(string? headerText, string message) + { + GuardAgainst.Null(message); + + return new(headerText, message); + } + + /// Creates an error. + public static ImageDownloadFailed CreateImageDownloadFailed(string imageUrl, string message) + { + GuardAgainst.Null(imageUrl); + GuardAgainst.Null(message); + + return new(imageUrl, message); + } + + /// Creates an error. + public static ImageSaveFailed CreateImageSaveFailed(string path, string message) + { + GuardAgainst.Null(path); + GuardAgainst.Null(message); + + return new(path, message); + } + + /// Creates a error. + public static ConfigurationSaveFailed CreateConfigurationSaveFailed(string message) + { + GuardAgainst.Null(message); + + return new(message); + } + + /// Creates a error. + public static ClassificationFailed CreateClassificationFailed(string fileName, string message) + { + GuardAgainst.Null(fileName); + GuardAgainst.Null(message); + + return new(fileName, message); + } + + /// Creates an error. + public static UnexpectedError CreateUnexpectedError(Exception exception) + { + GuardAgainst.Null(exception); + + return new(exception); + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorLoggingExtensions.cs b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorLoggingExtensions.cs new file mode 100644 index 0000000..fd49d78 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorLoggingExtensions.cs @@ -0,0 +1,12 @@ +using AStar.Dev.FunctionalParadigm; +using Serilog; + +namespace AStar.Dev.Wallpaper.Scrapper.Models; + +/// Logging extensions for pipelines that fail with a . +public static class ScrapeErrorLoggingExtensions +{ + /// Logs an error when is a failure, then returns unchanged. + public static Result LogFailure(this Result result, ILogger logger) + => result.Tap(_ => { }, error => logger.Error("Scrape pipeline failure: {Message}", error.Message)); +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Pages/ImageLinkSelector.cs b/src/AStar.Dev.Wallpaper.Scrapper/Pages/ImageLinkSelector.cs new file mode 100644 index 0000000..27034ae --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Pages/ImageLinkSelector.cs @@ -0,0 +1,11 @@ +namespace AStar.Dev.Wallpaper.Scrapper.Pages; + +/// Selects the wanted image links from a page's raw anchor href attributes. +public static class ImageLinkSelector +{ + /// Selects the non-null that point at an image, up to . + public static IReadOnlyCollection SelectWanted(IEnumerable hrefs) + => [.. hrefs.Where(href => href is not null && href.Contains("/w/", StringComparison.Ordinal)) + .Select(href => href!) + .Take(ScrapperConstants.ImagesPerPage)]; +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Pages/ImagePage.cs b/src/AStar.Dev.Wallpaper.Scrapper/Pages/ImagePage.cs index 2d4a2d6..3d9ea0a 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Pages/ImagePage.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Pages/ImagePage.cs @@ -3,6 +3,7 @@ using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Repositories; using AStar.Dev.Wallpaper.Scrapper.Services; +using AStar.Dev.Wallpaper.Scrapper.Support; using Microsoft.Playwright; namespace AStar.Dev.Wallpaper.Scrapper.Pages; @@ -17,120 +18,37 @@ public async Task GetImageFromPageAsync(string link, string cat _ = await page.GotoAsync(link); var tagLocators = await page.Locator(".tagname").AllAsync(); - string directoryName = scrapeConfiguration.ScrapeDirectories.BaseSaveDirectory.CombinePath(categoryName.Replace(' ', '-')); - var (directoryNameUpdated, filePrefix, skip, imageTags) = await ProcessTheImageTagsAsync(tagLocators, [directoryName]); - - if (skip) return new ImagePageResult(null, directoryNameUpdated, filePrefix, skip, imageTags); - - var imageTag = page.Locator("#wallpaper"); - string? sourcePath = await imageTag.GetAttributeAsync("src"); - - return new ImagePageResult(sourcePath, directoryNameUpdated, filePrefix, skip, imageTags); - } - - private async Task<(List directoryName, string filePrefix, bool skip, IReadOnlyList tags)> ProcessTheImageTagsAsync(IEnumerable tags, List directoryName) - { - bool skip = false; - string filePrefix = string.Empty; - - var tagData = await Task.WhenAll(tags.Select(GetTagsAsync)); - var imageTags = tagData.Select(t => t.Tag).Where(t => !string.IsNullOrWhiteSpace(t)).ToList(); + var tagData = await Task.WhenAll(tagLocators.Select(GetTagsAsync)); await scrapedTagRepository.SaveAsync([.. tagData.Where(t => !string.IsNullOrWhiteSpace(t.Category))]); - foreach (var (tagText, tagToUse) in tagData) - { - if (tagToUse == null) continue; - - string trimmedTagToUse = tagToUse.Trim(); - skip = IsOneOfTheImageTagsToExcludeCompletely(trimmedTagToUse) || IsOneOfTheImageTagsToExcludeCompletely(tagText); + string initialDirectory = scrapeConfiguration.ScrapeDirectories.BaseSaveDirectory.CombinePath(categoryName.Replace(' ', '-')); + var context = TagRuleContextFactory.Create(initialDirectory, scrapeConfiguration.ScrapeDirectories.BaseDirectoryFamous, tagsToIgnoreCompletely, tagsTextToIgnore); - if (skip) break; + var outcome = TagRules.Evaluate(tagData, context); - var (filePrefixUpdated, directoryNameUpdated) = UpdateFilePrefixForModels(trimmedTagToUse, tagText, filePrefix, directoryName); - - directoryName = directoryNameUpdated; - - filePrefix = UpdateFilePrefixForVehicles(trimmedTagToUse, filePrefixUpdated); - - if (UpdateToTagIsNotRequired(trimmedTagToUse, tagText, filePrefix)) continue; - - filePrefix = string.Join("-", filePrefix, tagText.Replace(' ', '-')).ToLowerInvariant(); - directoryName.Add(scrapeConfiguration.ScrapeDirectories.BaseDirectoryFamous); - } - - filePrefix = UpdateFilePrefixIfRequired(filePrefix); - - return (directoryName, filePrefix, skip, imageTags); + return await MapOutcomeToResultAsync(outcome); } - private static string UpdateFilePrefixIfRequired(string filePrefix) + private async Task MapOutcomeToResultAsync(TagOutcome outcome) + => outcome switch + { + SkipImage skip => new ImagePageResult(null, [], string.Empty, true, skip.Tags), + Accept accept => new ImagePageResult(await GetImageSourceAsync(), [.. accept.DirectorySegments], accept.FilePrefix, false, accept.Tags), + _ => throw new InvalidOperationException("Unexpected tag outcome."), + }; + + private async Task GetImageSourceAsync() { - if (filePrefix.StartsWith('-')) filePrefix = filePrefix[1..]; + var imageTag = page.Locator("#wallpaper"); - return filePrefix; + return await imageTag.GetAttributeAsync("src"); } - private bool UpdateToTagIsNotRequired(string tagToUse, string tagText, string filePrefix) - => TagIsNotCelebEtc(tagToUse) || FilePrefixDoesNotNeedUpdating(tagText, filePrefix); - private static async Task GetTagsAsync(ILocator tag) { string textTask = await tag.InnerTextAsync(); string? attrTask = await tag.GetAttributeAsync("original-title"); return new TagData(textTask, attrTask); } - - private bool FilePrefixDoesNotNeedUpdating(string tagText, string filePrefix) - => IsWantedText(tagText) || !filePrefix.Contains(tagText); - - private static bool TagIsNotCelebEtc(string tagToUse) - => !TagContains(tagToUse, "celeb") - && !TagContains(tagToUse, "singer") - && !TagContains(tagToUse, "actress"); - - private static bool TagContains(string tagToUse, string contains) - => tagToUse.Contains(contains, StringComparison.OrdinalIgnoreCase); - - private string UpdateFilePrefixForVehicles(string tagToUse, string filePrefix) - { - if (!TagContains(tagToUse, "Vehicles > Cars & Motorcycles")) return filePrefix; - - if (IsWantedFilePrefix(tagToUse, filePrefix)) filePrefix = string.Join("-", filePrefix, tagToUse); - - return filePrefix; - } - - private bool IsWantedFilePrefix(string tagToUse, string filePrefix) - => IsWantedText(tagToUse) && !filePrefix.Contains(tagToUse) && - !tagToUse.Equals("car", StringComparison.OrdinalIgnoreCase) && - !TagContains(tagToUse, "cars"); - - private (string filePrefix, List directoryName) UpdateFilePrefixForModels(string tagToUse, string tagText, string filePrefix, List directoryName) - { - string filePrefixUpdated = filePrefix; - - if (IsPeopleTag(tagToUse)) - { - if (IsWantedText(tagText) && !filePrefix.Contains(tagText)) - { - - if (!directoryName.Contains(tagText) && !filePrefix.Contains(tagText, StringComparison.OrdinalIgnoreCase)) - { - filePrefixUpdated = string.Join("-", filePrefix, tagText); - //directoryName.Add(tagText); - } - } - } - - return (filePrefixUpdated, directoryName); - } - - private static bool IsPeopleTag(string tagToUse) => TagContains(tagToUse, "people > model") || TagContains(tagToUse, "people > porn") || TagContains(tagToUse, "people > actress") || TagContains(tagToUse, "people > actor") || TagContains(tagToUse, "people > singer"); - - private bool IsOneOfTheImageTagsToExcludeCompletely(string tagText) - => tagsToIgnoreCompletely.Tags.Contains(tagText); - - private bool IsWantedText(string tagText) - => !tagsTextToIgnore.Tags.Contains(tagText) && !tagText.StartsWith("model", StringComparison.OrdinalIgnoreCase); } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Pages/PageHeaderParser.cs b/src/AStar.Dev.Wallpaper.Scrapper/Pages/PageHeaderParser.cs new file mode 100644 index 0000000..e7b06c0 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Pages/PageHeaderParser.cs @@ -0,0 +1,35 @@ +using System.Globalization; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Pages; + +/// Parses the header text shown on a search-results page into a . +public static class PageHeaderParser +{ + /// Parses , frozen from the historical SearchResultsPage.GetPageInfoAsync parsing quirks. + public static Result Parse(string? headerText) + { + if (string.IsNullOrEmpty(headerText)) + return ScrapeErrorFactory.CreatePageParseFailed(headerText, "The search results page header text was missing."); + + try + { + var firstSpaceIndex = headerText.IndexOf(' '); + var hashIndex = headerText.IndexOf("for ", StringComparison.Ordinal) + 3; + var subDirectoryName = string.Empty; + + if (hashIndex > 0) subDirectoryName = headerText[(hashIndex + 1)..].Replace(" ", "-").Replace("#", string.Empty); + + var searchResults = headerText.Replace(",", string.Empty)[..firstSpaceIndex]; + var imageCount = decimal.Parse(searchResults, CultureInfo.InvariantCulture); + var pageCount = Convert.ToInt32(Math.Ceiling(imageCount / ScrapperConstants.ImagesPerPage)); + + return PageInfoFactory.Create(pageCount, (int)imageCount, subDirectoryName); + } + catch (Exception exception) + { + return ScrapeErrorFactory.CreatePageParseFailed(headerText, exception.Message); + } + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Pages/SearchResultsPage.cs b/src/AStar.Dev.Wallpaper.Scrapper/Pages/SearchResultsPage.cs index e651361..5a07089 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Pages/SearchResultsPage.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Pages/SearchResultsPage.cs @@ -1,4 +1,4 @@ -using System.Globalization; +using AStar.Dev.FunctionalParadigm; using AStar.Dev.Wallpaper.Scrapper.Services; using Microsoft.Playwright; using Serilog.Core; @@ -71,17 +71,11 @@ public async Task> ImagePageLinksAsync() private async Task> GetTheImagePageLinksAsync() { - List wantedLinks = []; - var imagePreviews = await ImagePreviews.AllAsync(); + var imagePreviews = await ImagePreviews.AllAsync().ConfigureAwait(false); + List hrefs = []; + foreach (var imagePreview in imagePreviews) hrefs.Add(await imagePreview.GetAttributeAsync("href").ConfigureAwait(false)); - foreach (var imagePreview in imagePreviews) - { - string? hrefString = await imagePreview.GetAttributeAsync("href"); - - if (hrefString != null && hrefString.Contains("/w/")) wantedLinks.Add(hrefString); - } - - return [.. wantedLinks.Take(ScrapperConstants.ImagesPerPage)]; + return ImageLinkSelector.SelectWanted(hrefs); } private async Task<(int pageCount, int imageCount, string subDirectoryName)> GetPageInfoAsync() @@ -90,16 +84,9 @@ private async Task> GetTheImagePageLinksAsync() if (text is null) return (0, 0, string.Empty); - int firstSpaceIndex = text.IndexOf(' '); - int hashIndex = text.IndexOf("for ", StringComparison.Ordinal) + 3; - string subDirectoryName = string.Empty; - - if (hashIndex > 0) subDirectoryName = text[(hashIndex + 1)..].Replace(" ", "-").Replace("#", string.Empty); - - string searchResults = text.Replace(",", string.Empty)[..firstSpaceIndex]; - decimal imageCount = decimal.Parse(searchResults, CultureInfo.InvariantCulture); - - return (Convert.ToInt32(Math.Ceiling(imageCount / ScrapperConstants.ImagesPerPage)), (int)imageCount, subDirectoryName); + return PageHeaderParser.Parse(text).Match( + pageInfo => (pageInfo.PageCount, pageInfo.ImageCount, pageInfo.SubDirectoryName), + error => throw new InvalidOperationException(error.Message)); } private async Task GotoPageAsync(string searchString, int pageNumber) diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Pages/SubscriptionHeaderParser.cs b/src/AStar.Dev.Wallpaper.Scrapper/Pages/SubscriptionHeaderParser.cs new file mode 100644 index 0000000..5ac0907 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Pages/SubscriptionHeaderParser.cs @@ -0,0 +1,35 @@ +using System.Globalization; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Pages; + +/// Parses the header text shown on a subscriptions page into a . +public static class SubscriptionHeaderParser +{ + /// Parses , frozen from the historical SubscriptionsImagesListPage.PageInfoAsync parsing quirks. + public static Result Parse(string? headerText) + { + if (string.IsNullOrEmpty(headerText)) + return ScrapeErrorFactory.CreatePageParseFailed(headerText, "The subscriptions page header text was missing."); + + try + { + var firstSpaceIndex = headerText.IndexOf(' '); + var hashIndex = headerText.IndexOf("New", StringComparison.Ordinal); + var subDirectoryName = string.Empty; + + if (hashIndex > 0) subDirectoryName = headerText[hashIndex..].Replace(" ", "-").Replace("#", string.Empty); + + var searchResults = headerText.Replace(",", string.Empty)[..firstSpaceIndex]; + var imageCount = decimal.Parse(searchResults, CultureInfo.InvariantCulture); + var pageCount = Convert.ToInt32(Math.Ceiling(imageCount / ScrapperConstants.ImagesPerPage)); + + return PageInfoFactory.Create(pageCount, (int)imageCount, subDirectoryName); + } + catch (Exception exception) + { + return ScrapeErrorFactory.CreatePageParseFailed(headerText, exception.Message); + } + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Pages/SubscriptionsImagesListPage.cs b/src/AStar.Dev.Wallpaper.Scrapper/Pages/SubscriptionsImagesListPage.cs index d497ff9..7ab74fe 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Pages/SubscriptionsImagesListPage.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Pages/SubscriptionsImagesListPage.cs @@ -1,4 +1,4 @@ -using System.Globalization; +using AStar.Dev.FunctionalParadigm; using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Services; using Microsoft.Playwright; @@ -26,32 +26,19 @@ public sealed class SubscriptionsImagesListPage(IPlaywrightService playwrightSer if (text is null) return (0, string.Empty); - int firstSpaceIndex = text.IndexOf(' '); - int hashIndex = text.IndexOf("New", StringComparison.Ordinal); - string subDirectoryName = string.Empty; - - if (hashIndex > 0) subDirectoryName = text[hashIndex..].Replace(" ", "-").Replace("#", string.Empty); - - string searchResults = text.Replace(",", string.Empty)[..firstSpaceIndex]; - decimal imageCount = decimal.Parse(searchResults, CultureInfo.InvariantCulture) / ScrapperConstants.ImagesPerPage; - - return (Convert.ToInt32(Math.Ceiling(imageCount)), subDirectoryName); + return SubscriptionHeaderParser.Parse(text).Match( + pageInfo => (pageInfo.PageCount, pageInfo.SubDirectoryName), + error => throw new InvalidOperationException(error.Message)); } public async Task> GetImagePageLinksAsync() { page ??= await playwrightService.ConfigurePlaywrightAsync(); - List wantedLinks = []; - var imagePreviews = await ImagePreviews.AllAsync(); - - foreach (var imagePreview in imagePreviews) - { - string? hrefString = await imagePreview.GetAttributeAsync("href"); - - if (hrefString != null && hrefString.Contains("/w/")) wantedLinks.Add(hrefString); - } + var imagePreviews = await ImagePreviews.AllAsync().ConfigureAwait(false); + List hrefs = []; + foreach (var imagePreview in imagePreviews) hrefs.Add(await imagePreview.GetAttributeAsync("href").ConfigureAwait(false)); - return [.. wantedLinks.Take(ScrapperConstants.ImagesPerPage)]; + return ImageLinkSelector.SelectWanted(hrefs); } public async Task ClearAsync() diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Pages/TopWallpapersHeaderParser.cs b/src/AStar.Dev.Wallpaper.Scrapper/Pages/TopWallpapersHeaderParser.cs new file mode 100644 index 0000000..80ac31d --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Pages/TopWallpapersHeaderParser.cs @@ -0,0 +1,28 @@ +using System.Globalization; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Pages; + +/// Parses the header text shown on the top-wallpapers page into a total page count. +public static class TopWallpapersHeaderParser +{ + /// Parses , frozen from the historical TopWallpapersPage.PageInfoAsync parsing quirks. + public static Result Parse(string? headerText) + { + if (string.IsNullOrEmpty(headerText)) + return ScrapeErrorFactory.CreatePageParseFailed(headerText, "The top wallpapers page header text was missing."); + + try + { + var firstSlashIndex = headerText.IndexOf('/') + 1; + var pages = headerText[firstSlashIndex..].Trim(); + + return int.Parse(pages, CultureInfo.InvariantCulture); + } + catch (Exception exception) + { + return ScrapeErrorFactory.CreatePageParseFailed(headerText, exception.Message); + } + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Pages/TopWallpapersPage.cs b/src/AStar.Dev.Wallpaper.Scrapper/Pages/TopWallpapersPage.cs index c74ec71..7393ee8 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Pages/TopWallpapersPage.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Pages/TopWallpapersPage.cs @@ -1,4 +1,4 @@ -using System.Globalization; +using AStar.Dev.FunctionalParadigm; using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Services; using Microsoft.Playwright; @@ -26,25 +26,18 @@ public async Task PageInfoAsync() if (text is null) return 0; - int firstSlashIndex = text.IndexOf('/') + 1; - string pages = text[firstSlashIndex..].Trim(); - - return int.Parse(pages, CultureInfo.InvariantCulture); + return TopWallpapersHeaderParser.Parse(text).Match( + pageCount => pageCount, + error => throw new InvalidOperationException(error.Message)); } public async Task> GetImagePageLinksAsync() { page ??= await playwrightService.ConfigurePlaywrightAsync(); - List wantedLinks = []; - var imagePreviews = await ImagePreviews.AllAsync(); - - foreach (var imagePreview in imagePreviews) - { - string? hrefString = await imagePreview.GetAttributeAsync("href"); - - if (hrefString != null && hrefString.Contains("/w/")) wantedLinks.Add(hrefString); - } + var imagePreviews = await ImagePreviews.AllAsync().ConfigureAwait(false); + List hrefs = []; + foreach (var imagePreview in imagePreviews) hrefs.Add(await imagePreview.GetAttributeAsync("href").ConfigureAwait(false)); - return [.. wantedLinks.Take(ScrapperConstants.ImagesPerPage)]; + return ImageLinkSelector.SelectWanted(hrefs); } } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/ScrapperConstants.cs b/src/AStar.Dev.Wallpaper.Scrapper/ScrapperConstants.cs index ac1c9b8..89cd38b 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/ScrapperConstants.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/ScrapperConstants.cs @@ -11,4 +11,6 @@ public static class ScrapperConstants public static readonly TimeSpan PageNavigationDelay = TimeSpan.FromSeconds(2); public static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(10); + + public static readonly TimeSpan ImageAlreadyDownloadedDelay = TimeSpan.FromMilliseconds(500); } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Services/ImagePageService.cs b/src/AStar.Dev.Wallpaper.Scrapper/Services/ImagePageService.cs index 67b708d..20ac6c4 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Services/ImagePageService.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Services/ImagePageService.cs @@ -1,3 +1,5 @@ +using System.IO.Abstractions; +using AStar.Dev.FunctionalParadigm; using AStar.Dev.Infrastructure.AppDb.Entities; using AStar.Dev.Utilities; using AStar.Dev.Wallpaper.Scrapper.Models; @@ -9,8 +11,13 @@ namespace AStar.Dev.Wallpaper.Scrapper.Services; -public sealed class ImagePageService(ImagePage imagePage, IFileDetailRepository fileDetailRepository, FileClassificationService fileClassificationService, ScrapeConfiguration scrapeConfiguration, TimeProvider timeProvider, Logger logger, IDirectoryHelper directoryHelper, ImageBroadcaster imageBroadcaster) +public sealed class ImagePageService(ImagePage imagePage, IFileDetailRepository fileDetailRepository, FileClassificationService fileClassificationService, ScrapeConfiguration scrapeConfiguration, TimeProvider timeProvider, Logger logger, IDirectoryHelper directoryHelper, ImageBroadcaster imageBroadcaster, IDelayStrategy delayStrategy, IImageRetriever imageRetriever, IImageSaver imageSaver, IFileSystem fileSystem) { + private const int LoggedPathTailLength = 50; + + /// Retained for constructor-signature stability. The image-pause delay it previously drove now lives in , which is configured with its own . + private readonly ScrapeConfiguration scrapeConfiguration = scrapeConfiguration; + public async Task GetTheImagePagesAsync(IReadOnlyCollection imagePageLinks, string categoryId, string name, CancellationToken ct = default) { var pageData = await fileClassificationService.LoadPageClassificationDataAsync(categoryId, ct).ConfigureAwait(false); @@ -25,7 +32,7 @@ public async Task GetTheImagePagesAsync(IReadOnlyCollection imagePageLin if (await fileDetailRepository.ExistsAsync(fileName).ConfigureAwait(false)) { logger.Information("Not downloading {fileName} as we already have it...{Timestamp:HH:mm:ss:fff} (UTC)", fileName, timeProvider.GetUtcNow()); - await Task.Delay(TimeSpan.FromMilliseconds(500), ct).ConfigureAwait(false); + await delayStrategy.DelayAsync(DelayKind.ImageAlreadyDownloaded, ct).ConfigureAwait(false); continue; } @@ -34,7 +41,7 @@ public async Task GetTheImagePagesAsync(IReadOnlyCollection imagePageLin catch (Exception ex) when (ex is not OperationCanceledException) { logger.Warning(ex, "Failed to process {pageLink}, retrying after delay.", pageLink); - await Task.Delay(ScrapperConstants.RetryDelay, ct).ConfigureAwait(false); + await delayStrategy.DelayAsync(DelayKind.Retry, ct).ConfigureAwait(false); await ProcessImagePageAsync(pageLink, name, pageData, ct).ConfigureAwait(false); } } @@ -44,8 +51,7 @@ private async Task ProcessImagePageAsync(string pageLink, string name, PageClass { try { - int delay = Random.Shared.Next(scrapeConfiguration.SearchConfiguration.ImagePauseInSeconds, scrapeConfiguration.SearchConfiguration.ImagePauseInSeconds + 4); - await Task.Delay(TimeSpan.FromSeconds(delay), ct).ConfigureAwait(false); + await delayStrategy.DelayAsync(DelayKind.BeforeImage, ct).ConfigureAwait(false); var result = await imagePage.GetImageFromPageAsync(pageLink, name).ConfigureAwait(false); if (result.Skip || result.ImageUrl is null) @@ -59,12 +65,13 @@ private async Task ProcessImagePageAsync(string pageLink, string name, PageClass string filename = ScrapedFileNameFactory.Create(result.FilePrefix, result.ImageUrl); string imageNameWithPath = directoryName.Value.CombinePath(filename); - byte[] image = await ImageRetrieverHelper.GetTheImageAsync(result.ImageUrl).ConfigureAwait(false); - logger.Information("About to save {filename} to ...{imageNameWithPath} as we don't appear to have it.", filename, imageNameWithPath[^50..]); - await ImageSaveHelper.SaveImageAsync(image, imageNameWithPath).ConfigureAwait(false); + byte[] image = (await imageRetriever.GetImageAsync(result.ImageUrl, ct).ConfigureAwait(false)) + .Match(bytes => bytes, error => throw new InvalidOperationException(error.Message)); + logger.Information("About to save {filename} to ...{imageNameWithPath} as we don't appear to have it.", filename, TruncatedForLogging(imageNameWithPath)); + await imageSaver.SaveAsync(image, imageNameWithPath).ConfigureAwait(false); imageBroadcaster.Broadcast(imageNameWithPath); - var fileInfo = new FileInfo(imageNameWithPath); + var fileInfo = fileSystem.FileInfo.New(imageNameWithPath); var fileDetail = new FileDetailEntity { DirectoryName = directoryName, @@ -103,4 +110,7 @@ private async Task ProcessImagePageAsync(string pageLink, string name, PageClass throw; } } + + private static string TruncatedForLogging(string imageNameWithPath) + => imageNameWithPath.Length > LoggedPathTailLength ? imageNameWithPath[^LoggedPathTailLength..] : imageNameWithPath; } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/DelayKind.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/DelayKind.cs new file mode 100644 index 0000000..e19ec90 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/DelayKind.cs @@ -0,0 +1,20 @@ +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// Identifies which point in the scrape workflow an is being asked to delay for. +public enum DelayKind +{ + /// A search category was already up to date. + CategoryUpToDate, + + /// Navigating between search-result pages. + PageNavigation, + + /// An image was already downloaded. + ImageAlreadyDownloaded, + + /// Pausing before visiting an individual image page. + BeforeImage, + + /// Retrying a failed image page after a transient error. + Retry, +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/DirectoryHelper.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/DirectoryHelper.cs index e975f3a..fca43df 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Support/DirectoryHelper.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/DirectoryHelper.cs @@ -1,17 +1,23 @@ +using System.IO.Abstractions; +using AStar.Dev.Guard.Clauses; using AStar.Dev.Infrastructure.AppDb.Entities; using AStar.Dev.Utilities; namespace AStar.Dev.Wallpaper.Scrapper.Support; /// -public class DirectoryHelper : IDirectoryHelper +public sealed class DirectoryHelper(IFileSystem fileSystem) : IDirectoryHelper { /// public DirectoryName CreateDirectoryIfRequired(List fullDirectoryPath) { - string directory = Path.Combine([.. fullDirectoryPath])!; + GuardAgainst.Null(fullDirectoryPath); - _ = Directory.CreateDirectory(directory.CleanPath()); + if (fullDirectoryPath.Count == 0) return new(string.Empty); + + string directory = fullDirectoryPath[0].CombinePath([.. fullDirectoryPath.Skip(1),]); + + _ = fileSystem.Directory.CreateDirectory(directory.CleanPath()); return new(directory); } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/IDelayStrategy.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/IDelayStrategy.cs new file mode 100644 index 0000000..f700d55 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/IDelayStrategy.cs @@ -0,0 +1,8 @@ +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// Injects the non-deterministic delays a scrape workflow waits on, so the workflow itself stays testable without real waits. +public interface IDelayStrategy +{ + /// Delays for the duration appropriate to . + Task DelayAsync(DelayKind delayKind, CancellationToken cancellationToken = default); +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/IImageRetriever.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/IImageRetriever.cs new file mode 100644 index 0000000..1e22f2a --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/IImageRetriever.cs @@ -0,0 +1,11 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// Retrieves the raw bytes of a scraped image. +public interface IImageRetriever +{ + /// Downloads the image at . + Task> GetImageAsync(string url, CancellationToken cancellationToken = default); +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/IImageSaver.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/IImageSaver.cs new file mode 100644 index 0000000..09eb5ea --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/IImageSaver.cs @@ -0,0 +1,8 @@ +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// Saves a scraped image's bytes to disk. +public interface IImageSaver +{ + /// Saves to , cleaning the path first. + Task SaveAsync(byte[] image, string path); +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageRetriever.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageRetriever.cs new file mode 100644 index 0000000..c0d40e8 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageRetriever.cs @@ -0,0 +1,26 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// The production , backed by an injected . +public sealed class ImageRetriever(HttpClient httpClient) : IImageRetriever +{ + /// + public async Task> GetImageAsync(string url, CancellationToken cancellationToken = default) + { + try + { + var response = await httpClient.GetAsync(url, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + return ScrapeErrorFactory.CreateImageDownloadFailed(url, $"Received status code {(int)response.StatusCode} ({response.ReasonPhrase})."); + + return await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException exception) + { + return ScrapeErrorFactory.CreateImageDownloadFailed(url, exception.Message); + } + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageRetrieverHelper.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageRetrieverHelper.cs deleted file mode 100644 index e35484b..0000000 --- a/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageRetrieverHelper.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace AStar.Dev.Wallpaper.Scrapper.Support; - -internal static class ImageRetrieverHelper -{ - private static readonly HttpClient _client = new() { Timeout = TimeSpan.FromMinutes(2) }; - - public static async Task GetTheImageAsync(string src) - { - var response = await _client.GetAsync(src); - - return response is { IsSuccessStatusCode: true, } ? await response.Content.ReadAsByteArrayAsync() : []; - } -} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageSaveHelper.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageSaveHelper.cs deleted file mode 100644 index 350d795..0000000 --- a/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageSaveHelper.cs +++ /dev/null @@ -1,15 +0,0 @@ -using AStar.Dev.Utilities; - -namespace AStar.Dev.Wallpaper.Scrapper.Support; - -internal sealed class ImageSaveHelper -{ - public static async Task SaveImageAsync(byte[] image, string imageNameWithPath) - { - imageNameWithPath = imageNameWithPath.CleanPath(); - - if (imageNameWithPath.LastIndexOf(':') > 2) imageNameWithPath = imageNameWithPath[..2] + imageNameWithPath[2..].Replace(":", "_"); - - if (image.Length > 0) await File.WriteAllBytesAsync(imageNameWithPath, image); - } -} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageSaver.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageSaver.cs new file mode 100644 index 0000000..2b5931c --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageSaver.cs @@ -0,0 +1,18 @@ +using System.IO.Abstractions; +using AStar.Dev.Utilities; + +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// The production , backed by an injected , frozen from the historical ImageSaveHelper behaviour. +public sealed class ImageSaver(IFileSystem fileSystem) : IImageSaver +{ + /// + public async Task SaveAsync(byte[] image, string path) + { + string cleanedPath = path.CleanPath(); + + if (cleanedPath.LastIndexOf(':') > 2) cleanedPath = cleanedPath[..2] + cleanedPath[2..].Replace(":", "_"); + + if (image.Length > 0) await fileSystem.File.WriteAllBytesAsync(cleanedPath, image).ConfigureAwait(false); + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/RandomDelayStrategy.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/RandomDelayStrategy.cs new file mode 100644 index 0000000..5ffea6b --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/RandomDelayStrategy.cs @@ -0,0 +1,19 @@ +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// The production , using and . +public sealed class RandomDelayStrategy(ScrapeConfiguration scrapeConfiguration) : IDelayStrategy +{ + /// + public Task DelayAsync(DelayKind delayKind, CancellationToken cancellationToken = default) + => delayKind switch + { + DelayKind.CategoryUpToDate => Task.Delay(TimeSpan.FromSeconds(new Random().Next(1, 5)), cancellationToken), + DelayKind.PageNavigation => Task.Delay(ScrapperConstants.PageNavigationDelay, cancellationToken), + DelayKind.ImageAlreadyDownloaded => Task.Delay(ScrapperConstants.ImageAlreadyDownloadedDelay, cancellationToken), + DelayKind.BeforeImage => Task.Delay(TimeSpan.FromSeconds(Random.Shared.Next(scrapeConfiguration.SearchConfiguration.ImagePauseInSeconds, scrapeConfiguration.SearchConfiguration.ImagePauseInSeconds + 4)), cancellationToken), + DelayKind.Retry => Task.Delay(ScrapperConstants.RetryDelay, cancellationToken), + _ => throw new ArgumentOutOfRangeException(nameof(delayKind), delayKind, "Unrecognised delay kind."), + }; +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/TagOutcome.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/TagOutcome.cs new file mode 100644 index 0000000..677dab1 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/TagOutcome.cs @@ -0,0 +1,14 @@ +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// The outcome of evaluating an image's tags against the . +public abstract record TagOutcome; + +/// The image should be skipped because one of its tags matched an ignore-completely rule. +/// The tag text scraped from the image page before the skip was detected. +public sealed record SkipImage(IReadOnlyList Tags) : TagOutcome; + +/// The image should be kept, with the derived file prefix and directory segments to save it under. +/// The prefix to apply to the saved file name. +/// The directory segments the image should be saved under. +/// The tag text scraped from the image page. +public sealed record Accept(string FilePrefix, IReadOnlyList DirectorySegments, IReadOnlyList Tags) : TagOutcome; diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/TagOutcomeFactory.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/TagOutcomeFactory.cs new file mode 100644 index 0000000..a74e780 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/TagOutcomeFactory.cs @@ -0,0 +1,25 @@ +using AStar.Dev.Guard.Clauses; + +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// Factory methods for creating instances of the discriminated union. +public static class TagOutcomeFactory +{ + /// Creates a outcome. + public static SkipImage CreateSkipImage(IReadOnlyList tags) + { + GuardAgainst.Null(tags); + + return new(tags); + } + + /// Creates an outcome. + public static Accept CreateAccept(string filePrefix, IReadOnlyList directorySegments, IReadOnlyList tags) + { + GuardAgainst.Null(filePrefix); + GuardAgainst.Null(directorySegments); + GuardAgainst.Null(tags); + + return new(filePrefix, directorySegments, tags); + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/TagRuleContext.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/TagRuleContext.cs new file mode 100644 index 0000000..4702f3f --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/TagRuleContext.cs @@ -0,0 +1,10 @@ +using AStar.Dev.Wallpaper.Scrapper.DTOs; + +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// The configuration needs to evaluate an image's tags. +/// The directory segment the image is saved under before any tag rule is applied. +/// The directory segment added when a famous-person tag rule matches. +/// The tags/categories that cause an image to be skipped entirely. +/// The tag text that should not contribute to the derived file prefix. +public record TagRuleContext(string InitialDirectory, string BaseDirectoryFamous, TagsToIgnoreCompletely TagsToIgnoreCompletely, TagsTextToIgnore TagsTextToIgnore); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/TagRuleContextFactory.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/TagRuleContextFactory.cs new file mode 100644 index 0000000..0630d3f --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/TagRuleContextFactory.cs @@ -0,0 +1,19 @@ +using AStar.Dev.Guard.Clauses; +using AStar.Dev.Wallpaper.Scrapper.DTOs; + +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// Factory methods for creating instances of . +public static class TagRuleContextFactory +{ + /// Creates a . + public static TagRuleContext Create(string initialDirectory, string baseDirectoryFamous, TagsToIgnoreCompletely tagsToIgnoreCompletely, TagsTextToIgnore tagsTextToIgnore) + { + GuardAgainst.Null(initialDirectory); + GuardAgainst.Null(baseDirectoryFamous); + GuardAgainst.Null(tagsToIgnoreCompletely); + GuardAgainst.Null(tagsTextToIgnore); + + return new(initialDirectory, baseDirectoryFamous, tagsToIgnoreCompletely, tagsTextToIgnore); + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/TagRules.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/TagRules.cs new file mode 100644 index 0000000..634dc7a --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/TagRules.cs @@ -0,0 +1,92 @@ +using AStar.Dev.Wallpaper.Scrapper.Repositories; + +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// Pure classification rules for an image's tags, frozen from the historical ImagePage.ProcessTheImageTagsAsync behaviour. +public static class TagRules +{ + /// Evaluates against , deciding whether the image should be skipped or accepted. + public static TagOutcome Evaluate(IReadOnlyList tagData, TagRuleContext context) + { + var tags = ExtractTagText(tagData); + List directorySegments = [context.InitialDirectory,]; + string filePrefix = string.Empty; + + foreach (var (tagText, tagToUse) in tagData) + { + if (tagToUse is null) continue; + + string trimmedTagToUse = tagToUse.Trim(); + + if (IsOneOfTheImageTagsToExcludeCompletely(trimmedTagToUse, context) || IsOneOfTheImageTagsToExcludeCompletely(tagText, context)) + return TagOutcomeFactory.CreateSkipImage(tags); + + (filePrefix, directorySegments) = UpdateFilePrefixForModels(trimmedTagToUse, tagText, filePrefix, directorySegments, context); + filePrefix = UpdateFilePrefixForVehicles(trimmedTagToUse, filePrefix, context); + + if (UpdateToTagIsNotRequired(trimmedTagToUse, tagText, filePrefix, context)) continue; + + filePrefix = string.Join("-", filePrefix, tagText.Replace(' ', '-')).ToLowerInvariant(); + directorySegments = [.. directorySegments, context.BaseDirectoryFamous,]; + } + + filePrefix = StripLeadingDash(filePrefix); + + return TagOutcomeFactory.CreateAccept(filePrefix, directorySegments, tags); + } + + private static IReadOnlyList ExtractTagText(IReadOnlyList tagData) + => [.. tagData.Select(tag => tag.Tag).Where(tag => !string.IsNullOrWhiteSpace(tag)),]; + + private static string StripLeadingDash(string filePrefix) + => filePrefix.StartsWith('-') ? filePrefix[1..] : filePrefix; + + private static bool IsOneOfTheImageTagsToExcludeCompletely(string tagText, TagRuleContext context) + => context.TagsToIgnoreCompletely.Tags.Contains(tagText); + + private static bool IsWantedText(string tagText, TagRuleContext context) + => !context.TagsTextToIgnore.Tags.Contains(tagText) && !tagText.StartsWith("model", StringComparison.OrdinalIgnoreCase); + + private static bool TagContains(string tagToUse, string contains) + => tagToUse.Contains(contains, StringComparison.OrdinalIgnoreCase); + + private static bool IsPeopleTag(string tagToUse) + => TagContains(tagToUse, "people > model") + || TagContains(tagToUse, "people > porn") + || TagContains(tagToUse, "people > actress") + || TagContains(tagToUse, "people > actor") + || TagContains(tagToUse, "people > singer"); + + private static (string FilePrefix, List DirectorySegments) UpdateFilePrefixForModels(string tagToUse, string tagText, string filePrefix, List directorySegments, TagRuleContext context) + { + if (!IsPeopleTag(tagToUse)) return (filePrefix, directorySegments); + + if (!IsWantedText(tagText, context) || filePrefix.Contains(tagText)) return (filePrefix, directorySegments); + + if (directorySegments.Contains(tagText) || filePrefix.Contains(tagText, StringComparison.OrdinalIgnoreCase)) return (filePrefix, directorySegments); + + return (string.Join("-", filePrefix, tagText), directorySegments); + } + + private static string UpdateFilePrefixForVehicles(string tagToUse, string filePrefix, TagRuleContext context) + { + if (!TagContains(tagToUse, "Vehicles > Cars & Motorcycles")) return filePrefix; + + return IsWantedFilePrefix(tagToUse, filePrefix, context) ? string.Join("-", filePrefix, tagToUse) : filePrefix; + } + + private static bool IsWantedFilePrefix(string tagToUse, string filePrefix, TagRuleContext context) + => IsWantedText(tagToUse, context) + && !filePrefix.Contains(tagToUse) + && !tagToUse.Equals("car", StringComparison.OrdinalIgnoreCase) + && !TagContains(tagToUse, "cars"); + + private static bool UpdateToTagIsNotRequired(string tagToUse, string tagText, string filePrefix, TagRuleContext context) + => TagIsNotCelebEtc(tagToUse) || FilePrefixDoesNotNeedUpdating(tagText, filePrefix, context); + + private static bool TagIsNotCelebEtc(string tagToUse) + => !TagContains(tagToUse, "celeb") && !TagContains(tagToUse, "singer") && !TagContains(tagToUse, "actress"); + + private static bool FilePrefixDoesNotNeedUpdating(string tagText, string filePrefix, TagRuleContext context) + => IsWantedText(tagText, context) || !filePrefix.Contains(tagText); +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchProgress.cs b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchProgress.cs new file mode 100644 index 0000000..38350f1 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchProgress.cs @@ -0,0 +1,8 @@ +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Workflows; + +/// The mutable-in-name-only loop state threaded through a run. +/// The current search configuration. +/// The current scrape directories. +public record SearchProgress(SearchConfiguration SearchConfiguration, ScrapeDirectories ScrapeDirectories); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchProgressFactory.cs b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchProgressFactory.cs new file mode 100644 index 0000000..ec50ebe --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchProgressFactory.cs @@ -0,0 +1,17 @@ +using AStar.Dev.Guard.Clauses; +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Workflows; + +/// Factory methods for creating instances of . +public static class SearchProgressFactory +{ + /// Creates a . + public static SearchProgress Create(SearchConfiguration searchConfiguration, ScrapeDirectories scrapeDirectories) + { + GuardAgainst.Null(searchConfiguration); + GuardAgainst.Null(scrapeDirectories); + + return new(searchConfiguration, scrapeDirectories); + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchProgressFunctions.cs b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchProgressFunctions.cs new file mode 100644 index 0000000..0c65710 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchProgressFunctions.cs @@ -0,0 +1,46 @@ +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Workflows; + +/// Pure state transition functions for a , frozen from the historical SearchWorkflow private methods. +public static class SearchProgressFunctions +{ + /// Skips categories preceding the one matching the current search string, frozen from the historical FilterSearchCategories behaviour. + public static IReadOnlyList FilterSearchCategories(SearchConfiguration searchConfiguration, IReadOnlyList searchCategories) + { + for (int i = 0; i < searchCategories.Count; i++) + { + string combinedSearchString = $"{searchConfiguration.SearchStringPrefix}{searchCategories[i].Id}{searchConfiguration.SearchStringSuffix}"; + + if (combinedSearchString != searchConfiguration.SearchString) continue; + + return [.. searchCategories.Skip(i),]; + } + + return searchCategories; + } + + /// Updates the search string and resets the starting page number when the combined search string has changed. + public static SearchProgress UpdateSearchDetails(SearchProgress progress, string combinedSearchString) + { + if (progress.SearchConfiguration.SearchString == combinedSearchString) return progress; + + return progress with { SearchConfiguration = progress.SearchConfiguration with { StartingPageNumber = 1, SearchString = combinedSearchString, }, }; + } + + /// Updates the total page count when it has changed. + public static SearchProgress UpdateTotalPages(SearchProgress progress, int pageCount) + { + if (progress.SearchConfiguration.TotalPages == pageCount) return progress; + + return progress with { SearchConfiguration = progress.SearchConfiguration with { TotalPages = pageCount, }, }; + } + + /// Updates the sub-directory name when a non-empty name is supplied. + public static SearchProgress UpdateSubDirectory(SearchProgress progress, string subDirectoryName) + { + if (subDirectoryName.Length == 0) return progress; + + return progress with { ScrapeDirectories = progress.ScrapeDirectories with { SubDirectoryName = subDirectoryName, }, }; + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchWorkflow.cs b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchWorkflow.cs index c3bcd0b..e062582 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchWorkflow.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchWorkflow.cs @@ -1,29 +1,23 @@ -using System.Diagnostics; -using System.IO.Abstractions; using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Utilities; using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Pages; using AStar.Dev.Wallpaper.Scrapper.Services; using AStar.Dev.Wallpaper.Scrapper.Support; -using Microsoft.Playwright; using Serilog; namespace AStar.Dev.Wallpaper.Scrapper.Workflows; -public sealed class SearchWorkflow(SearchResultsPage searchResultsPage, ScrapeConfiguration injectedScrapeConfiguration, ConfigurationSaver configurationSaver, ImagePageService imagePageService, IDirectoryHelper directoryHelper, ILogger logger) +public sealed class SearchWorkflow(SearchResultsPage searchResultsPage, ScrapeConfiguration injectedScrapeConfiguration, ConfigurationSaver configurationSaver, ImagePageService imagePageService, IDirectoryHelper directoryHelper, ILogger logger, IDelayStrategy delayStrategy, TimeProvider timeProvider) { - private ScrapeConfiguration scrapeConfiguration = null!; - private SearchConfiguration searchConfiguration = null!; - private ScrapeDirectories scrapeDirectories = null!; + private SearchProgress progress = null!; public async Task> RunAsync(ILogger scrapeLogger, CancellationToken ct = default) { try { - scrapeConfiguration = injectedScrapeConfiguration; - searchConfiguration = scrapeConfiguration.SearchConfiguration; - scrapeDirectories = scrapeConfiguration.ScrapeDirectories; - var searchCategories = FilterSearchCategories([.. searchConfiguration.SearchCategories]); + progress = SearchProgressFactory.Create(injectedScrapeConfiguration.SearchConfiguration, injectedScrapeConfiguration.ScrapeDirectories); + var searchCategories = SearchProgressFunctions.FilterSearchCategories(progress.SearchConfiguration, progress.SearchConfiguration.SearchCategories); await ProcessSearchCategoriesAsync(searchCategories, scrapeLogger, ct); return Unit.Value; @@ -35,36 +29,36 @@ public async Task> RunAsync(ILogger scrapeLogger, Cancellat } } - private async Task ProcessSearchCategoriesAsync(List searchCategories, ILogger scrapeLogger, CancellationToken ct) + private async Task ProcessSearchCategoriesAsync(IReadOnlyList searchCategories, ILogger scrapeLogger, CancellationToken ct) { foreach (var searchCategory in searchCategories) { ct.ThrowIfCancellationRequested(); - string combinedSearchString = $"{searchConfiguration.SearchStringPrefix}{searchCategory.Id}{searchConfiguration.SearchStringSuffix}"; + string combinedSearchString = $"{progress.SearchConfiguration.SearchStringPrefix}{searchCategory.Id}{progress.SearchConfiguration.SearchStringSuffix}"; - searchConfiguration = UpdateSearchDetailsIfRequired(combinedSearchString); + progress = SearchProgressFunctions.UpdateSearchDetails(progress, combinedSearchString); - var pageDetails = await searchResultsPage.LoadSearchPageAsync(combinedSearchString, searchConfiguration.StartingPageNumber); + var pageDetails = await searchResultsPage.LoadSearchPageAsync(combinedSearchString, progress.SearchConfiguration.StartingPageNumber); if (pageDetails is { Ok: false, }) throw new InvalidOperationException("Could not get the image page after retry..."); var (pageCount, imageCount, subDirectoryName) = await searchResultsPage.PageInfoAsync(); - UpdateSearchTotalPagesIfRequired(pageCount); + progress = SearchProgressFunctions.UpdateTotalPages(progress, pageCount); if (searchCategory.IsUpToDate(imageCount, pageCount)) { logger.Information("{Category} is up to date (same image/page count), skipping...", searchCategory.Name); - await Task.Delay(TimeSpan.FromSeconds(RandomDelay()), ct); + await delayStrategy.DelayAsync(DelayKind.CategoryUpToDate, ct).ConfigureAwait(false); continue; } int startingPage = searchCategory.LastPageVisited > 0 ? searchCategory.LastPageVisited : 1; - searchConfiguration = searchConfiguration with { StartingPageNumber = startingPage }; + progress = progress with { SearchConfiguration = progress.SearchConfiguration with { StartingPageNumber = startingPage, }, }; logger.Debug("Visiting {Category} from page {StartingPage} now...", searchCategory.Name, startingPage); - scrapeDirectories = UpdateSubDirectoryIfRequired(subDirectoryName); + progress = SearchProgressFunctions.UpdateSubDirectory(progress, subDirectoryName); - _ = directoryHelper.CreateDirectoryIfRequired([Path.Combine(scrapeDirectories.RootDirectory, scrapeDirectories.BaseDirectory, subDirectoryName)]); + _ = directoryHelper.CreateDirectoryIfRequired([progress.ScrapeDirectories.RootDirectory.CombinePath(progress.ScrapeDirectories.BaseDirectory, subDirectoryName),]); await ProcessAllCategoryPagesAsync(searchCategory, combinedSearchString, scrapeLogger, ct); @@ -75,19 +69,16 @@ private async Task ProcessSearchCategoriesAsync(List searchCategories, } } - private static int RandomDelay() => new Random().Next(1, 5); - private async Task ProcessAllCategoryPagesAsync(Category searchCategory, string combinedSearchString, ILogger scrapeLogger, CancellationToken ct) { - var stopwatch = new Stopwatch(); - stopwatch.Start(); + long startTimestamp = timeProvider.GetTimestamp(); scrapeLogger.Debug("About to visit the specific {Category} pages now...", searchCategory.Name); - for (int currentPageNumber = searchConfiguration.StartingPageNumber; currentPageNumber <= searchConfiguration.TotalPages; currentPageNumber++) + for (int currentPageNumber = progress.SearchConfiguration.StartingPageNumber; currentPageNumber <= progress.SearchConfiguration.TotalPages; currentPageNumber++) { - await Task.Delay(ScrapperConstants.PageNavigationDelay, ct); - scrapeLogger.Debug("About to visit page {page} (of {totalPages}) for {Category} now...", currentPageNumber, searchConfiguration.TotalPages, searchCategory.Name); - searchConfiguration = searchConfiguration with { StartingPageNumber = currentPageNumber }; + await delayStrategy.DelayAsync(DelayKind.PageNavigation, ct).ConfigureAwait(false); + scrapeLogger.Debug("About to visit page {page} (of {totalPages}) for {Category} now...", currentPageNumber, progress.SearchConfiguration.TotalPages, searchCategory.Name); + progress = progress with { SearchConfiguration = progress.SearchConfiguration with { StartingPageNumber = currentPageNumber, }, }; searchCategory.LastPageVisited = currentPageNumber; await configurationSaver.SaveUpdatedConfigurationAsync(); _ = await searchResultsPage.LoadSearchPageAsync(combinedSearchString, currentPageNumber); @@ -96,43 +87,6 @@ private async Task ProcessAllCategoryPagesAsync(Category searchCategory, string await imagePageService.GetTheImagePagesAsync(imagePageLinks, searchCategory.Id, searchCategory.Name, ct); } - stopwatch.Stop(); - scrapeLogger.Information("Completed visiting the {Category}. Total time: {CategoryVisitDuration}", searchCategory.Name, stopwatch.Elapsed); - } - - private ScrapeDirectories UpdateSubDirectoryIfRequired(string subDirectoryName) - { - if (subDirectoryName.Length > 0) scrapeDirectories = scrapeDirectories with { SubDirectoryName = subDirectoryName }; - - return scrapeDirectories; - } - - private SearchConfiguration UpdateSearchDetailsIfRequired(string combinedSearchString) - { - if (searchConfiguration.SearchString == combinedSearchString) return searchConfiguration; - - searchConfiguration = searchConfiguration with { StartingPageNumber = 1, SearchString = combinedSearchString }; - - return searchConfiguration; - } - - private List FilterSearchCategories(List searchCategories) - { - for (int i = 0; i < searchCategories.Count; i++) - { - string combinedSearchString = $"{searchConfiguration.SearchStringPrefix}{searchCategories[i].Id}{searchConfiguration.SearchStringSuffix}"; - - if (combinedSearchString != searchConfiguration.SearchString) continue; - - searchCategories = [.. searchCategories.Skip(i)]; - break; - } - - return searchCategories; - } - - private void UpdateSearchTotalPagesIfRequired(int pageCount) - { - if (searchConfiguration.TotalPages != pageCount) searchConfiguration = searchConfiguration with { TotalPages = pageCount }; + scrapeLogger.Information("Completed visiting the {Category}. Total time: {CategoryVisitDuration}", searchCategory.Name, timeProvider.GetElapsedTime(startTimestamp)); } } diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/AStar.Dev.Wallpaper.Scrapper.Tests.Unit.csproj b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/AStar.Dev.Wallpaper.Scrapper.Tests.Unit.csproj index d3c2e22..1d46acc 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/AStar.Dev.Wallpaper.Scrapper.Tests.Unit.csproj +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/AStar.Dev.Wallpaper.Scrapper.Tests.Unit.csproj @@ -24,12 +24,22 @@ + + diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenTheLogFailureExtension.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenTheLogFailureExtension.cs new file mode 100644 index 0000000..0dc3bde --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenTheLogFailureExtension.cs @@ -0,0 +1,52 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; +using Serilog; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Models; + +public sealed class GivenTheLogFailureExtension +{ + [Fact] + public void when_the_result_is_a_success_then_the_logger_does_not_receive_an_error_call() + { + var logger = Substitute.For(); + var result = Result.Success("some value"); + + result.LogFailure(logger); + + logger.DidNotReceive().Error(Arg.Any(), Arg.Any()); + } + + [Fact] + public void when_the_result_is_a_success_then_the_same_result_is_returned() + { + var logger = Substitute.For(); + var result = Result.Success("some value"); + + var returned = result.LogFailure(logger); + + returned.ShouldBeSameAs(result); + } + + [Fact] + public void when_the_result_is_a_failure_then_the_logger_receives_exactly_one_error_call() + { + var logger = Substitute.For(); + var result = Result.Failure(ScrapeErrorFactory.CreateConfigurationSaveFailed("boom")); + + result.LogFailure(logger); + + logger.Received(1).Error(Arg.Any(), Arg.Any()); + } + + [Fact] + public void when_the_result_is_a_failure_then_the_same_result_is_returned() + { + var logger = Substitute.For(); + var result = Result.Failure(ScrapeErrorFactory.CreateConfigurationSaveFailed("boom")); + + var returned = result.LogFailure(logger); + + returned.ShouldBeSameAs(result); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenThePageInfoFactory.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenThePageInfoFactory.cs new file mode 100644 index 0000000..e819e30 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenThePageInfoFactory.cs @@ -0,0 +1,18 @@ +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Models; + +public sealed class GivenThePageInfoFactory +{ + [Fact] + public void when_creating_a_page_info_then_the_page_count_is_preserved() => + PageInfoFactory.Create(5, 120, "sub-directory").PageCount.ShouldBe(5); + + [Fact] + public void when_creating_a_page_info_then_the_image_count_is_preserved() => + PageInfoFactory.Create(5, 120, "sub-directory").ImageCount.ShouldBe(120); + + [Fact] + public void when_creating_a_page_info_then_the_sub_directory_name_is_preserved() => + PageInfoFactory.Create(5, 120, "sub-directory").SubDirectoryName.ShouldBe("sub-directory"); +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenTheScrapeErrorFactory.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenTheScrapeErrorFactory.cs new file mode 100644 index 0000000..dba15ca --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenTheScrapeErrorFactory.cs @@ -0,0 +1,50 @@ +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Models; + +public sealed class GivenTheScrapeErrorFactory +{ + [Fact] + public void when_creating_a_page_load_failed_error_then_the_url_is_preserved() => + ScrapeErrorFactory.CreatePageLoadFailed("https://example.test/page", "could not load").Url.ShouldBe("https://example.test/page"); + + [Fact] + public void when_creating_a_page_load_failed_error_then_the_message_is_preserved() => + ScrapeErrorFactory.CreatePageLoadFailed("https://example.test/page", "could not load").Message.ShouldBe("could not load"); + + [Fact] + public void when_creating_a_page_parse_failed_error_then_the_header_text_is_preserved() => + ScrapeErrorFactory.CreatePageParseFailed("some header", "could not parse").HeaderText.ShouldBe("some header"); + + [Fact] + public void when_creating_a_page_parse_failed_error_with_a_null_header_text_then_the_header_text_is_null() => + ScrapeErrorFactory.CreatePageParseFailed(null, "could not parse").HeaderText.ShouldBeNull(); + + [Fact] + public void when_creating_an_image_download_failed_error_then_the_image_url_is_preserved() => + ScrapeErrorFactory.CreateImageDownloadFailed("https://example.test/image.jpg", "download failed").ImageUrl.ShouldBe("https://example.test/image.jpg"); + + [Fact] + public void when_creating_an_image_save_failed_error_then_the_path_is_preserved() => + ScrapeErrorFactory.CreateImageSaveFailed("/some/path/image.jpg", "save failed").Path.ShouldBe("/some/path/image.jpg"); + + [Fact] + public void when_creating_a_configuration_save_failed_error_then_the_message_is_preserved() => + ScrapeErrorFactory.CreateConfigurationSaveFailed("configuration save failed").Message.ShouldBe("configuration save failed"); + + [Fact] + public void when_creating_a_classification_failed_error_then_the_file_name_is_preserved() => + ScrapeErrorFactory.CreateClassificationFailed("image.jpg", "classification failed").FileName.ShouldBe("image.jpg"); + + [Fact] + public void when_creating_an_unexpected_error_then_the_exception_is_preserved() + { + var exception = new InvalidOperationException("boom"); + + ScrapeErrorFactory.CreateUnexpectedError(exception).Exception.ShouldBeSameAs(exception); + } + + [Fact] + public void when_creating_an_unexpected_error_then_the_message_is_the_exception_message() => + ScrapeErrorFactory.CreateUnexpectedError(new InvalidOperationException("boom")).Message.ShouldBe("boom"); +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenTheImageLinkSelector.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenTheImageLinkSelector.cs new file mode 100644 index 0000000..b2937bb --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenTheImageLinkSelector.cs @@ -0,0 +1,39 @@ +using AStar.Dev.Wallpaper.Scrapper.Pages; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Pages; + +public sealed class GivenTheImageLinkSelector +{ + [Fact] + public void when_the_hrefs_are_empty_then_no_links_are_selected() => + ImageLinkSelector.SelectWanted([]).ShouldBeEmpty(); + + [Fact] + public void when_an_href_is_null_then_it_is_not_selected() => + ImageLinkSelector.SelectWanted([null, "https://example.test/w/1"]).ShouldBe(["https://example.test/w/1"]); + + [Fact] + public void when_an_href_does_not_contain_the_wanted_segment_then_it_is_not_selected() => + ImageLinkSelector.SelectWanted(["https://example.test/other/1", "https://example.test/w/1"]).ShouldBe(["https://example.test/w/1"]); + + [Fact] + public void when_there_are_more_than_the_images_per_page_limit_then_the_result_is_truncated_to_the_limit() + { + var hrefs = Enumerable.Range(1, 30).Select(i => $"https://example.test/w/{i}"); + + ImageLinkSelector.SelectWanted(hrefs).Count.ShouldBe(24); + } + + [Fact] + public void when_there_are_more_than_the_images_per_page_limit_then_the_first_matching_links_are_kept_in_order() + { + var hrefs = Enumerable.Range(1, 30).Select(i => $"https://example.test/w/{i}"); + + ImageLinkSelector.SelectWanted(hrefs).ShouldBe(Enumerable.Range(1, 24).Select(i => $"https://example.test/w/{i}")); + } + + [Fact] + public void when_the_wanted_hrefs_are_within_the_limit_then_their_order_is_preserved() => + ImageLinkSelector.SelectWanted(["https://example.test/w/3", "https://example.test/w/1", "https://example.test/w/2"]) + .ShouldBe(["https://example.test/w/3", "https://example.test/w/1", "https://example.test/w/2"]); +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenThePageHeaderParser.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenThePageHeaderParser.cs new file mode 100644 index 0000000..01e4527 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenThePageHeaderParser.cs @@ -0,0 +1,60 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Pages; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Pages; + +public sealed class GivenThePageHeaderParser +{ + [Fact] + public void when_the_header_text_is_null_then_a_page_parse_failed_error_is_returned() => + PageHeaderParser.Parse(null) + .ShouldBeOfType>() + .Error.ShouldBeOfType(); + + [Fact] + public void when_the_header_text_is_empty_then_a_page_parse_failed_error_is_returned() => + PageHeaderParser.Parse(string.Empty) + .ShouldBeOfType>() + .Error.ShouldBeOfType(); + + [Fact] + public void when_the_header_text_does_not_start_with_a_number_then_a_page_parse_failed_error_is_returned() => + PageHeaderParser.Parse("abc Wallpapers found for #bad") + .ShouldBeOfType>() + .Error.ShouldBeOfType(); + + [Fact] + public void when_the_image_count_has_two_commas_then_the_comma_stripped_slice_quirk_causes_a_page_parse_failed_error() => + PageHeaderParser.Parse("1,234,567 Wallpapers found for #huge") + .ShouldBeOfType>() + .Error.ShouldBeOfType(); + + [Theory] + [InlineData("1,234 Wallpapers found for #nature", 52, 1234, "nature")] + [InlineData("1 Wallpapers found for #x", 1, 1, "x")] + [InlineData("24 Wallpapers found for #Exact", 1, 24, "Exact")] + [InlineData("25 Wallpapers found for #Rounding", 2, 25, "Rounding")] + [InlineData("12,345 Wallpapers found for #medium", 515, 12345, "medium")] + public void when_the_header_text_is_parseable_then_the_page_info_is_correctly_derived(string headerText, int expectedPageCount, int expectedImageCount, string expectedSubDirectoryName) + { + var pageInfo = PageHeaderParser.Parse(headerText).ShouldBeOfType>().Value; + + pageInfo.PageCount.ShouldBe(expectedPageCount); + pageInfo.ImageCount.ShouldBe(expectedImageCount); + pageInfo.SubDirectoryName.ShouldBe(expectedSubDirectoryName); + } + + [Fact] + public void when_the_header_has_no_for_keyword_then_the_sub_directory_name_reflects_the_absent_for_quirk() => + PageHeaderParser.Parse("500 Wallpapers found") + .ShouldBeOfType>() + .Value.SubDirectoryName.ShouldBe("-Wallpapers-found"); + + [Theory] + [InlineData("10,000 Wallpapers found for #Big Category", "Big-Category")] + public void when_the_sub_directory_name_contains_spaces_and_a_hash_then_they_are_replaced(string headerText, string expectedSubDirectoryName) => + PageHeaderParser.Parse(headerText) + .ShouldBeOfType>() + .Value.SubDirectoryName.ShouldBe(expectedSubDirectoryName); +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenTheSubscriptionHeaderParser.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenTheSubscriptionHeaderParser.cs new file mode 100644 index 0000000..e89c1b7 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenTheSubscriptionHeaderParser.cs @@ -0,0 +1,45 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Pages; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Pages; + +public sealed class GivenTheSubscriptionHeaderParser +{ + [Fact] + public void when_the_header_text_is_null_then_a_page_parse_failed_error_is_returned() => + SubscriptionHeaderParser.Parse(null) + .ShouldBeOfType>() + .Error.ShouldBeOfType(); + + [Fact] + public void when_the_header_text_is_empty_then_a_page_parse_failed_error_is_returned() => + SubscriptionHeaderParser.Parse(string.Empty) + .ShouldBeOfType>() + .Error.ShouldBeOfType(); + + [Fact] + public void when_the_header_text_does_not_start_with_a_number_then_a_page_parse_failed_error_is_returned() => + SubscriptionHeaderParser.Parse("abc New Subscription Wallpapers") + .ShouldBeOfType>() + .Error.ShouldBeOfType(); + + [Theory] + [InlineData("48 New Subscription Wallpapers", 2, 48, "New-Subscription-Wallpapers")] + [InlineData("1 New Subscription Wallpapers", 1, 1, "New-Subscription-Wallpapers")] + [InlineData("1,000 New Subscription Wallpapers", 42, 1000, "New-Subscription-Wallpapers")] + public void when_the_header_text_is_parseable_then_the_page_info_is_correctly_derived(string headerText, int expectedPageCount, int expectedImageCount, string expectedSubDirectoryName) + { + var pageInfo = SubscriptionHeaderParser.Parse(headerText).ShouldBeOfType>().Value; + + pageInfo.PageCount.ShouldBe(expectedPageCount); + pageInfo.ImageCount.ShouldBe(expectedImageCount); + pageInfo.SubDirectoryName.ShouldBe(expectedSubDirectoryName); + } + + [Fact] + public void when_the_header_has_no_new_keyword_then_the_sub_directory_name_is_empty() => + SubscriptionHeaderParser.Parse("48 Something Else") + .ShouldBeOfType>() + .Value.SubDirectoryName.ShouldBe(string.Empty); +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenTheTopWallpapersHeaderParser.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenTheTopWallpapersHeaderParser.cs new file mode 100644 index 0000000..618b8da --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenTheTopWallpapersHeaderParser.cs @@ -0,0 +1,38 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Pages; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Pages; + +public sealed class GivenTheTopWallpapersHeaderParser +{ + [Fact] + public void when_the_header_text_is_null_then_a_page_parse_failed_error_is_returned() => + TopWallpapersHeaderParser.Parse(null) + .ShouldBeOfType>() + .Error.ShouldBeOfType(); + + [Fact] + public void when_the_header_text_is_empty_then_a_page_parse_failed_error_is_returned() => + TopWallpapersHeaderParser.Parse(string.Empty) + .ShouldBeOfType>() + .Error.ShouldBeOfType(); + + [Fact] + public void when_the_header_text_is_garbage_then_a_page_parse_failed_error_is_returned() => + TopWallpapersHeaderParser.Parse("garbage text") + .ShouldBeOfType>() + .Error.ShouldBeOfType(); + + [Fact] + public void when_the_header_text_has_no_slash_then_a_page_parse_failed_error_is_returned() => + TopWallpapersHeaderParser.Parse("Page 5") + .ShouldBeOfType>() + .Error.ShouldBeOfType(); + + [Theory] + [InlineData("Page 3 / 42", 42)] + [InlineData("Page 1 / 1", 1)] + public void when_the_header_text_is_parseable_then_the_total_page_count_is_returned(string headerText, int expectedPageCount) => + TopWallpapersHeaderParser.Parse(headerText).ShouldBeOfType>().Value.ShouldBe(expectedPageCount); +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImagePageServiceWithADelayStrategy.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImagePageServiceWithADelayStrategy.cs new file mode 100644 index 0000000..35b37d0 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImagePageServiceWithADelayStrategy.cs @@ -0,0 +1,158 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Infrastructure.AppDb; +using AStar.Dev.Infrastructure.AppDb.Entities; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Pages; +using AStar.Dev.Wallpaper.Scrapper.Repositories; +using AStar.Dev.Wallpaper.Scrapper.Support; +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Playwright; +using Serilog; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Services; + +public sealed class GivenAnImagePageServiceWithADelayStrategy : IAsyncLifetime +{ + private const string Link = "https://example.test/w/12345"; + private const string CategoryName = "Cars"; + private const string CategoryId = "cat-1"; + private const string ImageUrl = "https://example.test/images/12345.data"; + + private SqliteConnection connection = null!; + private DbContextOptions options = null!; + + public async ValueTask InitializeAsync() + { + connection = new SqliteConnection("Data Source=:memory:"); + await connection.OpenAsync(); + + options = new DbContextOptionsBuilder().UseSqlite(connection).Options; + + await using var seedContext = new AppDbContext(options); + await seedContext.Database.MigrateAsync(); + } + + public async ValueTask DisposeAsync() => await connection.DisposeAsync(); + + private static ImagePage BuildImagePageReturningImage(string imageUrl) + { + var tagLocator = Substitute.For(); + tagLocator.InnerTextAsync().Returns(Task.FromResult("Nature")); + tagLocator.GetAttributeAsync("original-title").Returns(Task.FromResult("Landscape")); + + var tagsLocator = Substitute.For(); + tagsLocator.AllAsync().Returns(Task.FromResult>([tagLocator,])); + + var imageLocator = Substitute.For(); + imageLocator.GetAttributeAsync("src").Returns(Task.FromResult(imageUrl)); + + var page = Substitute.For(); + page.Locator(".tagname", Arg.Any()).Returns(tagsLocator); + page.Locator("#wallpaper", Arg.Any()).Returns(imageLocator); + + var playwrightService = Substitute.For(); + playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + + var scrapeConfiguration = new ScrapeConfigurationBuilder().Build(); + + return new ImagePage(playwrightService, scrapeConfiguration, new(), new(), Substitute.For()); + } + + private ImagePageService BuildService(ImagePage imagePage, IFileDetailRepository fileDetailRepository, IDelayStrategy delayStrategy, IImageRetriever imageRetriever, IImageSaver imageSaver, MockFileSystem fileSystem, IDirectoryHelper directoryHelper) + { + var contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); + var fileClassificationService = new FileClassificationService(contextFactory); + var scrapeConfiguration = new ScrapeConfigurationBuilder().Build(); + + return new ImagePageService(imagePage, fileDetailRepository, fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), directoryHelper, new(), delayStrategy, imageRetriever, imageSaver, fileSystem); + } + + [Fact] + public async Task when_the_file_already_exists_then_the_delay_strategy_receives_the_image_already_downloaded_delay() + { + var fileDetailRepository = Substitute.For(); + fileDetailRepository.ExistsAsync(Arg.Any()).Returns(true); + var delayStrategy = Substitute.For(); + delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + var imagePage = BuildImagePageReturningImage(ImageUrl); + + var sut = BuildService(imagePage, fileDetailRepository, delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For()); + + await sut.GetTheImagePagesAsync([Link,], CategoryId, CategoryName, TestContext.Current.CancellationToken); + + await delayStrategy.Received(1).DelayAsync(DelayKind.ImageAlreadyDownloaded, Arg.Any()); + } + + [Fact] + public async Task when_the_file_does_not_already_exist_then_the_delay_strategy_receives_the_before_image_delay() + { + var fileDetailRepository = Substitute.For(); + fileDetailRepository.ExistsAsync(Arg.Any()).Returns(false); + var delayStrategy = Substitute.For(); + delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + var imagePage = BuildImagePageReturningImage(ImageUrl); + var directoryHelper = Substitute.For(); + directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); + var imageRetriever = Substitute.For(); + imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result.Success([1, 2, 3,]))); + var imageSaver = Substitute.For(); + var fileSystem = new MockFileSystem(); + fileSystem.Directory.CreateDirectory("/save/dir"); + fileSystem.File.WriteAllBytes("/save/dir/12345.data", [1, 2, 3,]); + + var sut = BuildService(imagePage, fileDetailRepository, delayStrategy, imageRetriever, imageSaver, fileSystem, directoryHelper); + + await sut.GetTheImagePagesAsync([Link,], CategoryId, CategoryName, TestContext.Current.CancellationToken); + + await delayStrategy.Received(1).DelayAsync(DelayKind.BeforeImage, Arg.Any()); + } + + [Fact] + public async Task when_the_image_retriever_always_fails_then_get_the_image_pages_async_retries_exactly_once_before_throwing() + { + var fileDetailRepository = Substitute.For(); + fileDetailRepository.ExistsAsync(Arg.Any()).Returns(false); + var delayStrategy = Substitute.For(); + delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + var imagePage = BuildImagePageReturningImage(ImageUrl); + var directoryHelper = Substitute.For(); + directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); + var imageRetriever = Substitute.For(); + imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(Result.Failure(ScrapeErrorFactory.CreateImageDownloadFailed(ImageUrl, "download failed")))); + var imageSaver = Substitute.For(); + var fileSystem = new MockFileSystem(); + + var sut = BuildService(imagePage, fileDetailRepository, delayStrategy, imageRetriever, imageSaver, fileSystem, directoryHelper); + + await Should.ThrowAsync(() => sut.GetTheImagePagesAsync([Link,], CategoryId, CategoryName, TestContext.Current.CancellationToken)); + + await imageRetriever.Received(2).GetImageAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task when_the_image_retriever_always_fails_then_the_delay_strategy_receives_exactly_one_retry_delay() + { + var fileDetailRepository = Substitute.For(); + fileDetailRepository.ExistsAsync(Arg.Any()).Returns(false); + var delayStrategy = Substitute.For(); + delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + var imagePage = BuildImagePageReturningImage(ImageUrl); + var directoryHelper = Substitute.For(); + directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); + var imageRetriever = Substitute.For(); + imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(Result.Failure(ScrapeErrorFactory.CreateImageDownloadFailed(ImageUrl, "download failed")))); + var imageSaver = Substitute.For(); + var fileSystem = new MockFileSystem(); + + var sut = BuildService(imagePage, fileDetailRepository, delayStrategy, imageRetriever, imageSaver, fileSystem, directoryHelper); + + await Should.ThrowAsync(() => sut.GetTheImagePagesAsync([Link,], CategoryId, CategoryName, TestContext.Current.CancellationToken)); + + await delayStrategy.Received(1).DelayAsync(DelayKind.Retry, Arg.Any()); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheDirectoryHelperWithAFileSystem.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheDirectoryHelperWithAFileSystem.cs new file mode 100644 index 0000000..d22d535 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheDirectoryHelperWithAFileSystem.cs @@ -0,0 +1,36 @@ +using AStar.Dev.Wallpaper.Scrapper.Support; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Support; + +public sealed class GivenTheDirectoryHelperWithAFileSystem +{ + private readonly MockFileSystem fileSystem = new(); + private readonly IDirectoryHelper sut; + + public GivenTheDirectoryHelperWithAFileSystem() => sut = new DirectoryHelper(fileSystem); + + [Fact] + public void when_the_directory_does_not_exist_then_it_is_created() + { + sut.CreateDirectoryIfRequired(["root", "sub-directory",]); + + fileSystem.Directory.Exists(Path.Combine("root", "sub-directory")).ShouldBeTrue(); + } + + [Fact] + public void when_the_directory_is_created_then_the_returned_directory_name_is_the_combined_pre_clean_path() + { + var directoryName = sut.CreateDirectoryIfRequired(["root", "sub-directory",]); + + directoryName.Value.ShouldBe(Path.Combine("root", "sub-directory")); + } + + [Fact] + public void when_the_combined_path_contains_characters_requiring_cleaning_then_the_directory_is_created_at_the_cleaned_path_but_the_returned_name_keeps_the_uncleaned_path() + { + var directoryName = sut.CreateDirectoryIfRequired(["root", "sub\"directory",]); + + fileSystem.Directory.Exists(Path.Combine("root", "sub'directory")).ShouldBeTrue(); + directoryName.Value.ShouldBe(Path.Combine("root", "sub\"directory")); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheImageRetriever.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheImageRetriever.cs new file mode 100644 index 0000000..9ebd67a --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheImageRetriever.cs @@ -0,0 +1,51 @@ +using System.Net; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Support; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Support; + +public sealed class GivenTheImageRetriever +{ + private const string ImageUrl = "https://example.test/images/12345.jpg"; + + [Fact] + public async Task when_the_response_is_successful_then_the_image_bytes_are_returned() + { + byte[] expectedBytes = [1, 2, 3, 4,]; + var handler = new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(expectedBytes), }); + var sut = new ImageRetriever(new HttpClient(handler)); + + var bytes = (await sut.GetImageAsync(ImageUrl, TestContext.Current.CancellationToken)).ShouldBeOfType>().Value; + + bytes.ShouldBe(expectedBytes); + } + + [Fact] + public async Task when_the_response_is_not_successful_then_an_image_download_failed_error_is_returned() + { + var handler = new StubHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.NotFound)); + var sut = new ImageRetriever(new HttpClient(handler)); + + var error = (await sut.GetImageAsync(ImageUrl, TestContext.Current.CancellationToken)).ShouldBeOfType>().Error.ShouldBeOfType(); + + error.ImageUrl.ShouldBe(ImageUrl); + } + + [Fact] + public async Task when_the_http_client_throws_a_request_exception_then_an_image_download_failed_error_is_returned() + { + var handler = new StubHttpMessageHandler(_ => throw new HttpRequestException("connection refused")); + var sut = new ImageRetriever(new HttpClient(handler)); + + var error = (await sut.GetImageAsync(ImageUrl, TestContext.Current.CancellationToken)).ShouldBeOfType>().Error.ShouldBeOfType(); + + error.ImageUrl.ShouldBe(ImageUrl); + } +} + +internal sealed class StubHttpMessageHandler(Func responder) : HttpMessageHandler +{ + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + => Task.FromResult(responder(request)); +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheImageSaver.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheImageSaver.cs new file mode 100644 index 0000000..367a2c6 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheImageSaver.cs @@ -0,0 +1,54 @@ +using AStar.Dev.Wallpaper.Scrapper.Support; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Support; + +public sealed class GivenTheImageSaver +{ + private readonly MockFileSystem fileSystem = new(); + private readonly IImageSaver sut; + + public GivenTheImageSaver() => sut = new ImageSaver(fileSystem); + + [Fact] + public async Task when_the_image_is_empty_then_no_file_is_written() + { + fileSystem.Directory.CreateDirectory("/save/dir"); + + await sut.SaveAsync([], "/save/dir/image.jpg"); + + fileSystem.File.Exists("/save/dir/image.jpg").ShouldBeFalse(); + } + + [Fact] + public async Task when_the_image_is_not_empty_then_the_file_is_written_with_the_supplied_bytes() + { + fileSystem.Directory.CreateDirectory("/save/dir"); + byte[] image = [1, 2, 3,]; + + await sut.SaveAsync(image, "/save/dir/image.jpg"); + + fileSystem.File.ReadAllBytes("/save/dir/image.jpg").ShouldBe(image); + } + + [Fact] + public async Task when_the_path_contains_a_colon_after_the_second_character_then_it_is_replaced_with_an_underscore() + { + fileSystem.Directory.CreateDirectory("/tmp"); + byte[] image = [1,]; + + await sut.SaveAsync(image, "/tmp/save:name.jpg"); + + fileSystem.File.Exists("/tmp/save_name.jpg").ShouldBeTrue(); + } + + [Fact] + public async Task when_the_path_contains_a_quote_then_it_is_replaced_with_a_single_quote() + { + fileSystem.Directory.CreateDirectory("/save/dir"); + byte[] image = [1,]; + + await sut.SaveAsync(image, "/save/dir/file\"name.jpg"); + + fileSystem.File.Exists("/save/dir/file'name.jpg").ShouldBeTrue(); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheRandomDelayStrategy.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheRandomDelayStrategy.cs new file mode 100644 index 0000000..38a5c6b --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheRandomDelayStrategy.cs @@ -0,0 +1,23 @@ +using AStar.Dev.Wallpaper.Scrapper.Support; +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Support; + +public sealed class GivenTheRandomDelayStrategy +{ + private readonly IDelayStrategy sut = new RandomDelayStrategy(new ScrapeConfigurationBuilder().Build()); + + [Theory] + [InlineData(DelayKind.CategoryUpToDate)] + [InlineData(DelayKind.PageNavigation)] + [InlineData(DelayKind.ImageAlreadyDownloaded)] + [InlineData(DelayKind.BeforeImage)] + [InlineData(DelayKind.Retry)] + public async Task when_the_cancellation_token_is_already_cancelled_then_delay_async_throws(DelayKind delayKind) + { + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + + await Should.ThrowAsync(() => sut.DelayAsync(delayKind, cts.Token)); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheTagRuleContextFactory.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheTagRuleContextFactory.cs new file mode 100644 index 0000000..2370f82 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheTagRuleContextFactory.cs @@ -0,0 +1,26 @@ +using AStar.Dev.Wallpaper.Scrapper.DTOs; +using AStar.Dev.Wallpaper.Scrapper.Support; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Support; + +public sealed class GivenTheTagRuleContextFactory +{ + private static readonly TagsToIgnoreCompletely IgnoreCompletely = new(); + private static readonly TagsTextToIgnore TextToIgnore = new(); + + [Fact] + public void when_creating_a_context_then_the_initial_directory_is_preserved() => + TagRuleContextFactory.Create("initial-directory", "famous-directory", IgnoreCompletely, TextToIgnore).InitialDirectory.ShouldBe("initial-directory"); + + [Fact] + public void when_creating_a_context_then_the_base_directory_famous_is_preserved() => + TagRuleContextFactory.Create("initial-directory", "famous-directory", IgnoreCompletely, TextToIgnore).BaseDirectoryFamous.ShouldBe("famous-directory"); + + [Fact] + public void when_creating_a_context_then_the_tags_to_ignore_completely_is_preserved() => + TagRuleContextFactory.Create("initial-directory", "famous-directory", IgnoreCompletely, TextToIgnore).TagsToIgnoreCompletely.ShouldBeSameAs(IgnoreCompletely); + + [Fact] + public void when_creating_a_context_then_the_tags_text_to_ignore_is_preserved() => + TagRuleContextFactory.Create("initial-directory", "famous-directory", IgnoreCompletely, TextToIgnore).TagsTextToIgnore.ShouldBeSameAs(TextToIgnore); +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheTagRules.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheTagRules.cs new file mode 100644 index 0000000..d3c07d5 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheTagRules.cs @@ -0,0 +1,172 @@ +using AStar.Dev.Wallpaper.Scrapper.DTOs; +using AStar.Dev.Wallpaper.Scrapper.Repositories; +using AStar.Dev.Wallpaper.Scrapper.Support; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Support; + +public sealed class GivenTheTagRules +{ + private const string InitialDirectory = "initial"; + private const string BaseDirectoryFamous = "famous"; + + private static TagRuleContext Context(IEnumerable? ignoreCompletely = null, IEnumerable? textToIgnore = null) + => TagRuleContextFactory.Create( + InitialDirectory, + BaseDirectoryFamous, + new TagsToIgnoreCompletely { Tags = [.. ignoreCompletely ?? []], }, + new TagsTextToIgnore { Tags = [.. textToIgnore ?? []], }); + + [Fact] + public void when_there_are_no_tags_then_the_outcome_is_accepted_with_an_empty_prefix_and_only_the_initial_directory() + { + var outcome = TagRules.Evaluate([], Context()).ShouldBeOfType(); + + outcome.FilePrefix.ShouldBe(string.Empty); + outcome.DirectorySegments.ShouldBe([InitialDirectory,]); + outcome.Tags.ShouldBeEmpty(); + } + + [Fact] + public void when_a_tag_category_matches_the_ignore_completely_list_then_the_image_is_skipped() + { + List tags = [new("Sports Car", "Vehicles > Cars & Motorcycles"), new("BadCategory", "ExcludedCategory"),]; + + var outcome = TagRules.Evaluate(tags, Context(ignoreCompletely: ["ExcludedCategory",])).ShouldBeOfType(); + + outcome.Tags.ShouldBe(["Sports Car", "BadCategory",]); + } + + [Fact] + public void when_a_tag_text_matches_the_ignore_completely_list_then_the_image_is_skipped() + { + List tags = [new("BadTagText", "SomeCategory"),]; + + var outcome = TagRules.Evaluate(tags, Context(ignoreCompletely: ["BadTagText",])).ShouldBeOfType(); + + outcome.Tags.ShouldBe(["BadTagText",]); + } + + [Fact] + public void when_a_skip_match_occurs_mid_list_then_later_tags_are_not_processed_but_the_outcome_is_still_skipped() + { + List tags = [new("Sports Car", "Vehicles > Cars & Motorcycles"), new("BadCategory", "ExcludedCategory"), new("LaterTag", "Vehicles > Cars & Motorcycles"),]; + + var outcome = TagRules.Evaluate(tags, Context(ignoreCompletely: ["ExcludedCategory",])).ShouldBeOfType(); + + outcome.Tags.ShouldBe(["Sports Car", "BadCategory", "LaterTag",]); + } + + [Fact] + public void when_a_tag_has_a_null_category_then_it_is_excluded_from_rule_processing_but_its_text_is_kept_in_tags() + { + List tags = [new("JustAText", null),]; + + var outcome = TagRules.Evaluate(tags, Context()).ShouldBeOfType(); + + outcome.FilePrefix.ShouldBe(string.Empty); + outcome.Tags.ShouldBe(["JustAText",]); + } + + [Fact] + public void when_a_people_model_category_tag_text_is_wanted_then_the_file_prefix_gains_the_tag_text() + { + List tags = [new("Britney Spears", "People > Model"),]; + + var outcome = TagRules.Evaluate(tags, Context()).ShouldBeOfType(); + + outcome.FilePrefix.ShouldBe("Britney Spears"); + } + + [Fact] + public void when_a_people_model_category_tag_text_is_in_the_tags_text_to_ignore_list_then_the_file_prefix_is_not_updated() + { + List tags = [new("Britney Spears", "People > Model"),]; + + var outcome = TagRules.Evaluate(tags, Context(textToIgnore: ["Britney Spears",])).ShouldBeOfType(); + + outcome.FilePrefix.ShouldBe(string.Empty); + } + + [Fact] + public void when_a_people_model_category_tag_text_starts_with_model_then_it_is_treated_as_unwanted_text_and_the_file_prefix_is_not_updated() + { + List tags = [new("model Agency", "People > Model"),]; + + var outcome = TagRules.Evaluate(tags, Context()).ShouldBeOfType(); + + outcome.FilePrefix.ShouldBe(string.Empty); + } + + [Fact] + public void when_a_duplicate_people_model_tag_text_appears_twice_then_the_file_prefix_is_not_duplicated() + { + List tags = [new("Famous Person", "People > Model"), new("Famous Person", "People > Model"),]; + + var outcome = TagRules.Evaluate(tags, Context()).ShouldBeOfType(); + + outcome.FilePrefix.ShouldBe("Famous Person"); + } + + [Fact] + public void when_a_tag_category_contains_vehicles_cars_and_motorcycles_then_the_file_prefix_is_never_updated() + { + List tags = [new("AnyText", "Anything Vehicles > Cars & Motorcycles Anything"),]; + + var outcome = TagRules.Evaluate(tags, Context()).ShouldBeOfType(); + + outcome.FilePrefix.ShouldBe(string.Empty); + } + + [Fact] + public void when_a_tag_category_is_exactly_car_then_the_vehicles_branch_does_not_trigger_and_the_file_prefix_is_not_updated() + { + List tags = [new("AnyText", "car"),]; + + var outcome = TagRules.Evaluate(tags, Context()).ShouldBeOfType(); + + outcome.FilePrefix.ShouldBe(string.Empty); + } + + [Fact] + public void when_a_celeb_related_category_tag_text_is_unwanted_and_already_contained_in_the_prefix_then_it_is_appended_lowercased_with_dashes_and_the_base_directory_famous_is_added() + { + List tags = [new("Actress Extraordinaire", "People > Actress"), new("Actress", "Some > Actress Category"),]; + + var outcome = TagRules.Evaluate(tags, Context(textToIgnore: ["Actress",])).ShouldBeOfType(); + + outcome.FilePrefix.ShouldBe("actress extraordinaire-actress"); + outcome.DirectorySegments.ShouldBe([InitialDirectory, BaseDirectoryFamous,]); + } + + [Fact] + public void when_a_singer_related_category_tag_text_is_unwanted_and_already_contained_in_the_prefix_then_it_is_appended_lowercased_with_dashes() + { + List tags = [new("Singer Extraordinaire", "People > Singer"), new("Singer", "Some > Singer Category"),]; + + var outcome = TagRules.Evaluate(tags, Context(textToIgnore: ["Singer",])).ShouldBeOfType(); + + outcome.FilePrefix.ShouldBe("singer extraordinaire-singer"); + } + + [Fact] + public void when_the_final_file_prefix_would_start_with_a_dash_then_the_leading_dash_is_stripped() + { + List tags = [new("Britney Spears", "People > Model"),]; + + var outcome = TagRules.Evaluate(tags, Context()).ShouldBeOfType(); + + outcome.FilePrefix.ShouldStartWith("Britney"); + outcome.FilePrefix.ShouldNotStartWith("-"); + } + + [Fact] + public void when_there_is_a_mix_of_wanted_null_and_empty_tags_then_only_non_whitespace_tag_text_is_kept_in_tags() + { + List tags = [new("Nature", "Landscape"), new("Britney Spears", "People > Model"), new(string.Empty, "EmptyTagText"), new("Some Tag", null),]; + + var outcome = TagRules.Evaluate(tags, Context()).ShouldBeOfType(); + + outcome.Tags.ShouldBe(["Nature", "Britney Spears", "Some Tag",]); + outcome.FilePrefix.ShouldBe("Britney Spears"); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/TestData/NoOpDelayStrategy.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/TestData/NoOpDelayStrategy.cs new file mode 100644 index 0000000..902e2c9 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/TestData/NoOpDelayStrategy.cs @@ -0,0 +1,8 @@ +using AStar.Dev.Wallpaper.Scrapper.Support; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; + +internal sealed class NoOpDelayStrategy : IDelayStrategy +{ + public Task DelayAsync(DelayKind delayKind, CancellationToken cancellationToken = default) => Task.CompletedTask; +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithADelayStrategy.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithADelayStrategy.cs new file mode 100644 index 0000000..70b33e1 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithADelayStrategy.cs @@ -0,0 +1,171 @@ +using System.Diagnostics; +using AStar.Dev.Infrastructure.AppDb; +using AStar.Dev.Infrastructure.AppDb.Entities; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Pages; +using AStar.Dev.Wallpaper.Scrapper.Repositories; +using AStar.Dev.Wallpaper.Scrapper.Services; +using AStar.Dev.Wallpaper.Scrapper.Support; +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; +using AStar.Dev.Wallpaper.Scrapper.Workflows; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Playwright; +using Serilog; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Workflows; + +public sealed class GivenASearchWorkflowWithADelayStrategy : IAsyncLifetime +{ + private const string SearchStringPrefix = "https://example.test/search/"; + private const string SearchStringSuffix = "/?page="; + private const string CategoryId = "cat-delay"; + + private SqliteConnection connection = null!; + private DbContextOptions options = null!; + + public async ValueTask InitializeAsync() + { + connection = new SqliteConnection("Data Source=:memory:"); + await connection.OpenAsync(); + + options = new DbContextOptionsBuilder().UseSqlite(connection).Options; + + await using var seedContext = new AppDbContext(options); + await seedContext.Database.MigrateAsync(); + + seedContext.ScrapeConfiguration.Add(new ScrapeConfigurationEntity + { + ConnectionStrings = new ConnectionStringsEntity { Sqlite = "Data Source=test.db", }, + UserConfiguration = new UserConfigurationEntity { LoginEmailAddress = "user@example.com", Username = "user", Password = "password", SessionCookie = "cookie", }, + SearchConfiguration = new SearchConfigurationEntity { BaseUrl = new Uri("https://example.com"), }, + ScrapeDirectories = new ScrapeDirectoriesEntity { RootDirectory = "root-directory", }, + }); + await seedContext.SaveChangesAsync(); + } + + public async ValueTask DisposeAsync() => await connection.DisposeAsync(); + + private static (IPage Page, IPlaywrightService PlaywrightService) BuildPlaywright(string headerText, bool includeLinks) + { + var okResponse = Substitute.For(); + okResponse.Ok.Returns(true); + + var page = Substitute.For(); + page.GotoAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(okResponse)); + + var headerLocator = Substitute.For(); + headerLocator.TextContentAsync().Returns(Task.FromResult(headerText)); + page.GetByText(Arg.Is(text => text.Contains("Wallpapers found")), Arg.Any()).Returns(headerLocator); + + var linksLocator = Substitute.For(); + linksLocator.AllAsync().Returns(Task.FromResult>(includeLinks ? [] : [])); + page.GetByRole(AriaRole.Link, Arg.Any()).Returns(linksLocator); + + var playwrightService = Substitute.For(); + playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + + return (page, playwrightService); + } + + [Fact] + public async Task when_a_category_is_up_to_date_then_the_delay_strategy_receives_the_category_up_to_date_delay() + { + var category = new Category { Id = CategoryId, Name = "Delay Category", LastKnownImageCount = 10, TotalPages = 1, }; + string matchingSearchString = $"{SearchStringPrefix}{category.Id}{SearchStringSuffix}"; + var (_, playwrightService) = BuildPlaywright("10 Wallpapers found for #Delay", includeLinks: false); + + var searchConfiguration = new SearchConfigurationBuilder + { + SearchCategories = [category,], + SearchString = matchingSearchString, + SearchStringPrefix = SearchStringPrefix, + SearchStringSuffix = SearchStringSuffix, + }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + + var contextFactory = Substitute.For>(); + var searchResultsPage = new SearchResultsPage(playwrightService, new LoggerConfiguration().CreateLogger()); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new(), Substitute.For()); + var fileClassificationService = new FileClassificationService(contextFactory); + var delayStrategy = Substitute.For(); + delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem()); + + var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), delayStrategy, System.TimeProvider.System); + + await sut.RunAsync(Substitute.For(), TestContext.Current.CancellationToken); + + await delayStrategy.Received(1).DelayAsync(DelayKind.CategoryUpToDate, Arg.Any()); + } + + [Fact] + public async Task when_a_category_is_not_up_to_date_then_the_delay_strategy_receives_a_page_navigation_delay_for_each_page_visited() + { + var category = new Category { Id = CategoryId, Name = "Delay Category", LastKnownImageCount = 999, TotalPages = 999, }; + string matchingSearchString = $"{SearchStringPrefix}{category.Id}{SearchStringSuffix}"; + var (_, playwrightService) = BuildPlaywright("48 Wallpapers found for #Delay", includeLinks: false); + + var searchConfiguration = new SearchConfigurationBuilder + { + SearchCategories = [category,], + SearchString = matchingSearchString, + SearchStringPrefix = SearchStringPrefix, + SearchStringSuffix = SearchStringSuffix, + }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + + var contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); + + var searchResultsPage = new SearchResultsPage(playwrightService, new LoggerConfiguration().CreateLogger()); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new(), Substitute.For()); + var fileClassificationService = new FileClassificationService(contextFactory); + var delayStrategy = Substitute.For(); + delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem()); + + var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), delayStrategy, System.TimeProvider.System); + + await sut.RunAsync(Substitute.For(), TestContext.Current.CancellationToken); + + await delayStrategy.Received(2).DelayAsync(DelayKind.PageNavigation, Arg.Any()); + } + + [Fact] + public async Task when_the_delay_strategy_is_a_zero_delay_substitute_then_run_async_completes_almost_instantly() + { + var category = new Category { Id = CategoryId, Name = "Delay Category", LastKnownImageCount = 999, TotalPages = 999, }; + string matchingSearchString = $"{SearchStringPrefix}{category.Id}{SearchStringSuffix}"; + var (_, playwrightService) = BuildPlaywright("48 Wallpapers found for #Delay", includeLinks: false); + + var searchConfiguration = new SearchConfigurationBuilder + { + SearchCategories = [category,], + SearchString = matchingSearchString, + SearchStringPrefix = SearchStringPrefix, + SearchStringSuffix = SearchStringSuffix, + }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + + var contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); + + var searchResultsPage = new SearchResultsPage(playwrightService, new LoggerConfiguration().CreateLogger()); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new(), Substitute.For()); + var fileClassificationService = new FileClassificationService(contextFactory); + var delayStrategy = new NoOpDelayStrategy(); + var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem()); + + var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), delayStrategy, System.TimeProvider.System); + + var stopwatch = Stopwatch.StartNew(); + await sut.RunAsync(Substitute.For(), TestContext.Current.CancellationToken); + stopwatch.Stop(); + + stopwatch.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(2)); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithAMidListSearchString.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithAMidListSearchString.cs index 0ed8f77..3efb8da 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithAMidListSearchString.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithAMidListSearchString.cs @@ -54,9 +54,9 @@ public async Task when_run_then_categories_before_the_matching_category_are_not_ var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new(), Substitute.For()); var fileClassificationService = new FileClassificationService(contextFactory); - var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new()); + var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), new NoOpDelayStrategy(), Substitute.For(), Substitute.For(), new MockFileSystem()); - var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For()); + var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), new NoOpDelayStrategy(), System.TimeProvider.System); await sut.RunAsync(Substitute.For(), TestContext.Current.CancellationToken); diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithANonEmptySubDirectory.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithANonEmptySubDirectory.cs index 4c7d544..072b48a 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithANonEmptySubDirectory.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithANonEmptySubDirectory.cs @@ -85,10 +85,10 @@ public async Task when_the_page_reports_a_non_empty_sub_directory_name_then_the_ var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new(), Substitute.For()); var fileClassificationService = new FileClassificationService(contextFactory); - var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new()); + var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), new NoOpDelayStrategy(), Substitute.For(), Substitute.For(), new MockFileSystem()); var directoryHelper = Substitute.For(); - var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, directoryHelper, Substitute.For()); + var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, directoryHelper, Substitute.For(), new NoOpDelayStrategy(), System.TimeProvider.System); await sut.RunAsync(Substitute.For(), TestContext.Current.CancellationToken); diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenTheSearchProgressFactory.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenTheSearchProgressFactory.cs new file mode 100644 index 0000000..e7c7546 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenTheSearchProgressFactory.cs @@ -0,0 +1,25 @@ +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; +using AStar.Dev.Wallpaper.Scrapper.Workflows; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Workflows; + +public sealed class GivenTheSearchProgressFactory +{ + [Fact] + public void when_creating_a_search_progress_then_the_search_configuration_is_preserved() + { + var searchConfiguration = new SearchConfigurationBuilder().Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + + SearchProgressFactory.Create(searchConfiguration, scrapeConfiguration.ScrapeDirectories).SearchConfiguration.ShouldBeSameAs(searchConfiguration); + } + + [Fact] + public void when_creating_a_search_progress_then_the_scrape_directories_is_preserved() + { + var searchConfiguration = new SearchConfigurationBuilder().Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + + SearchProgressFactory.Create(searchConfiguration, scrapeConfiguration.ScrapeDirectories).ScrapeDirectories.ShouldBeSameAs(scrapeConfiguration.ScrapeDirectories); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenTheSearchProgressFunctions.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenTheSearchProgressFunctions.cs new file mode 100644 index 0000000..8b86498 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenTheSearchProgressFunctions.cs @@ -0,0 +1,130 @@ +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; +using AStar.Dev.Wallpaper.Scrapper.Workflows; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Workflows; + +public sealed class GivenTheSearchProgressFunctions +{ + private const string SearchStringPrefix = "https://example.test/search/"; + private const string SearchStringSuffix = "/?page="; + + private static SearchProgress Progress(SearchConfiguration searchConfiguration) + => SearchProgressFactory.Create(searchConfiguration, new ScrapeConfigurationBuilder().Build().ScrapeDirectories); + + [Fact] + public void when_no_category_matches_the_current_search_string_then_the_whole_list_is_returned() + { + var categoryOne = new Category { Id = "one", }; + var categoryTwo = new Category { Id = "two", }; + var searchConfiguration = new SearchConfigurationBuilder { SearchString = "no-match", SearchStringPrefix = SearchStringPrefix, SearchStringSuffix = SearchStringSuffix, }.Build(); + + var result = SearchProgressFunctions.FilterSearchCategories(searchConfiguration, [categoryOne, categoryTwo,]); + + result.ShouldBe([categoryOne, categoryTwo,]); + } + + [Fact] + public void when_a_category_mid_list_matches_the_current_search_string_then_preceding_categories_are_skipped() + { + var categoryOne = new Category { Id = "one", }; + var categoryTwo = new Category { Id = "two", }; + var categoryThree = new Category { Id = "three", }; + string matchingSearchString = $"{SearchStringPrefix}two{SearchStringSuffix}"; + var searchConfiguration = new SearchConfigurationBuilder { SearchString = matchingSearchString, SearchStringPrefix = SearchStringPrefix, SearchStringSuffix = SearchStringSuffix, }.Build(); + + var result = SearchProgressFunctions.FilterSearchCategories(searchConfiguration, [categoryOne, categoryTwo, categoryThree,]); + + result.ShouldBe([categoryTwo, categoryThree,]); + } + + [Fact] + public void when_the_first_category_matches_the_current_search_string_then_the_whole_list_is_returned() + { + var categoryOne = new Category { Id = "one", }; + var categoryTwo = new Category { Id = "two", }; + string matchingSearchString = $"{SearchStringPrefix}one{SearchStringSuffix}"; + var searchConfiguration = new SearchConfigurationBuilder { SearchString = matchingSearchString, SearchStringPrefix = SearchStringPrefix, SearchStringSuffix = SearchStringSuffix, }.Build(); + + var result = SearchProgressFunctions.FilterSearchCategories(searchConfiguration, [categoryOne, categoryTwo,]); + + result.ShouldBe([categoryOne, categoryTwo,]); + } + + [Fact] + public void when_the_combined_search_string_equals_the_current_search_string_then_the_progress_is_unchanged() + { + var searchConfiguration = new SearchConfigurationBuilder { SearchString = "same-string", StartingPageNumber = 7, }.Build(); + var progress = Progress(searchConfiguration); + + var result = SearchProgressFunctions.UpdateSearchDetails(progress, "same-string"); + + result.SearchConfiguration.StartingPageNumber.ShouldBe(7); + } + + [Fact] + public void when_the_combined_search_string_differs_from_the_current_search_string_then_the_search_string_is_updated() + { + var searchConfiguration = new SearchConfigurationBuilder { SearchString = "old-string", }.Build(); + var progress = Progress(searchConfiguration); + + var result = SearchProgressFunctions.UpdateSearchDetails(progress, "new-string"); + + result.SearchConfiguration.SearchString.ShouldBe("new-string"); + } + + [Fact] + public void when_the_combined_search_string_differs_from_the_current_search_string_then_the_starting_page_number_is_reset_to_one() + { + var searchConfiguration = new SearchConfigurationBuilder { SearchString = "old-string", StartingPageNumber = 9, }.Build(); + var progress = Progress(searchConfiguration); + + var result = SearchProgressFunctions.UpdateSearchDetails(progress, "new-string"); + + result.SearchConfiguration.StartingPageNumber.ShouldBe(1); + } + + [Fact] + public void when_the_page_count_equals_the_current_total_pages_then_the_progress_is_unchanged() + { + var searchConfiguration = new SearchConfigurationBuilder { TotalPages = 3, }.Build(); + var progress = Progress(searchConfiguration); + + var result = SearchProgressFunctions.UpdateTotalPages(progress, 3); + + result.SearchConfiguration.TotalPages.ShouldBe(3); + } + + [Fact] + public void when_the_page_count_differs_from_the_current_total_pages_then_the_total_pages_is_updated() + { + var searchConfiguration = new SearchConfigurationBuilder { TotalPages = 3, }.Build(); + var progress = Progress(searchConfiguration); + + var result = SearchProgressFunctions.UpdateTotalPages(progress, 8); + + result.SearchConfiguration.TotalPages.ShouldBe(8); + } + + [Fact] + public void when_the_sub_directory_name_is_empty_then_the_progress_is_unchanged() + { + var scrapeDirectories = new ScrapeConfigurationBuilder().Build().ScrapeDirectories; + var progress = SearchProgressFactory.Create(new SearchConfigurationBuilder().Build(), scrapeDirectories); + + var result = SearchProgressFunctions.UpdateSubDirectory(progress, string.Empty); + + result.ScrapeDirectories.SubDirectoryName.ShouldBe(scrapeDirectories.SubDirectoryName); + } + + [Fact] + public void when_the_sub_directory_name_is_not_empty_then_the_scrape_directories_sub_directory_name_is_updated() + { + var scrapeDirectories = new ScrapeConfigurationBuilder().Build().ScrapeDirectories; + var progress = SearchProgressFactory.Create(new SearchConfigurationBuilder().Build(), scrapeDirectories); + + var result = SearchProgressFunctions.UpdateSubDirectory(progress, "new-sub-directory"); + + result.ScrapeDirectories.SubDirectoryName.ShouldBe("new-sub-directory"); + } +}