diff --git a/src/AStar.Dev.FunctionalParadigm/Exceptional{T}.cs b/src/AStar.Dev.FunctionalParadigm/Exceptional{T}.cs index 5bfa75c..653410c 100644 --- a/src/AStar.Dev.FunctionalParadigm/Exceptional{T}.cs +++ b/src/AStar.Dev.FunctionalParadigm/Exceptional{T}.cs @@ -8,6 +8,14 @@ namespace AStar.Dev.FunctionalParadigm; /// The type of the success value. public abstract record Exceptional { + /// + /// Restricts derivation of to and + /// , both declared in this assembly. + /// + private protected Exceptional() + { + } + /// /// Implicitly lifts a success value into an . /// diff --git a/src/AStar.Dev.FunctionalParadigm/Result{TResult,TError}.cs b/src/AStar.Dev.FunctionalParadigm/Result{TResult,TError}.cs index 7006ad5..7e71b1a 100644 --- a/src/AStar.Dev.FunctionalParadigm/Result{TResult,TError}.cs +++ b/src/AStar.Dev.FunctionalParadigm/Result{TResult,TError}.cs @@ -9,6 +9,14 @@ namespace AStar.Dev.FunctionalParadigm; /// The type of the failure error. public abstract record Result { + /// + /// Restricts derivation of to and + /// , both declared in this assembly. + /// + private protected Result() + { + } + /// /// Implicitly lifts a success value into a . /// diff --git a/src/AStar.Dev.FunctionalParadigm/RetryExtensions.cs b/src/AStar.Dev.FunctionalParadigm/RetryExtensions.cs new file mode 100644 index 0000000..a64232e --- /dev/null +++ b/src/AStar.Dev.FunctionalParadigm/RetryExtensions.cs @@ -0,0 +1,31 @@ +namespace AStar.Dev.FunctionalParadigm; + +/// +/// Retry helpers for -returning operations. +/// +public static class RetryExtensions +{ + /// + /// Runs once. If it fails, runs and then runs + /// a second and final time, returning whichever attempt's result is current + /// at that point (the second attempt's result when both fail). + /// + /// The operation to run, at most twice. + /// The side effect (typically a delay) to run between the first and second attempts. + public static async Task> RetryOnceAsync(Func>> operation, Func onRetry) + { + ArgumentNullException.ThrowIfNull(operation); + ArgumentNullException.ThrowIfNull(onRetry); + + var firstAttempt = await operation().ConfigureAwait(false); + + return await firstAttempt.MatchAsync( + value => Task.FromResult(Result.Success(value)), + async _ => + { + await onRetry().ConfigureAwait(false); + + return await operation().ConfigureAwait(false); + }).ConfigureAwait(false); + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs b/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs index 30dfbed..2167ade 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs @@ -98,14 +98,14 @@ public override void OnFrameworkInitializationCompleted() .AddTransient() .AddSingleton() .AddSingleton() - .AddTransient() - .AddHttpClient(client => client.Timeout = TimeSpan.FromMinutes(2)) - .Services - .AddTransient(_ => TimeProvider.System) - .AddTransient>(sp => () => sp.GetRequiredService()) - .AddTransient>(sp => () => sp.GetRequiredService()) .AddTransient>(sp => () => sp.GetRequiredService()) - .AddTransient(); + .AddTransient>(sp => () => sp.GetRequiredService()) + .AddTransient() + .AddTransient>(sp => () => sp.GetRequiredService()) + .AddTransient(_ => TimeProvider.System) + .AddTransient() + .AddTransient() + .AddHttpClient(client => client.Timeout = TimeSpan.FromMinutes(2)); _host = builder.Build(); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Classifications/ClassificationsView.axaml.cs b/src/AStar.Dev.Wallpaper.Scrapper/Classifications/ClassificationsView.axaml.cs index ecd7afc..ee11cbd 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Classifications/ClassificationsView.axaml.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Classifications/ClassificationsView.axaml.cs @@ -1,4 +1,5 @@ using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Services; using AStar.Dev.Wallpaper.Scrapper.Support; using Avalonia.Controls; @@ -55,7 +56,7 @@ private async void OnImportClicked(object? sender, RoutedEventArgs e) } ) .Tap(_ => logger.Information("Importing classifications...")) - .Bind(_ => importExportService.ImportFileClassificationsFromFile()) + .Bind(_ => importExportService.ImportFileClassificationsFromFile().ToStringError()) .MapAsync(classifications => fileClassificationService.ImportClassificationsAsync(classifications, cts!.Token)) .TapAsync(_ => logger.Information("Import completed...")) .EnsureAsync(() => ResetUI()); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/DTOs/FileClassificationDtoValidator.cs b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/FileClassificationDtoValidator.cs new file mode 100644 index 0000000..7bbb818 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/DTOs/FileClassificationDtoValidator.cs @@ -0,0 +1,41 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Guard.Clauses; + +namespace AStar.Dev.Wallpaper.Scrapper.DTOs; + +/// Validates an imported row, accumulating every problem found instead of stopping at the first. +public static class FileClassificationDtoValidator +{ + private const int MinimumLevel = 1; + private const int MaximumLevel = 3; + + /// Validates , tagging any errors with its for reporting. + public static Validation Validate(FileClassification dto, int rowIndex) + { + GuardAgainst.Null(dto); + + Validation>> constructor = Validation.Valid>>(_ => _ => dto); + + return constructor + .Apply(ValidateName(dto, rowIndex)) + .Apply(ValidateLevel(dto, rowIndex)); + } + + /// Validates every row in , accumulating every invalid row's errors, in row order. + public static Validation> ValidateAll(IReadOnlyList dtos) + { + GuardAgainst.Null(dtos); + + return dtos.Select(Validate).Combine(); + } + + private static Validation ValidateName(FileClassification dto, int rowIndex) + => string.IsNullOrWhiteSpace(dto.Name) + ? Validation.Invalid(ValidationErrorFactory.Create($"Categories[{rowIndex}].Name", "Name is required.")) + : Validation.Valid(dto.Name); + + private static Validation ValidateLevel(FileClassification dto, int rowIndex) + => dto.Level is < MinimumLevel or > MaximumLevel + ? Validation.Invalid(ValidationErrorFactory.Create($"Categories[{rowIndex}].Level", $"Level must be between {MinimumLevel} and {MaximumLevel}.")) + : Validation.Valid(dto.Level); +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/MainWindow.axaml.cs b/src/AStar.Dev.Wallpaper.Scrapper/MainWindow.axaml.cs index 76426a4..a6e653c 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/MainWindow.axaml.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/MainWindow.axaml.cs @@ -1,6 +1,7 @@ using AStar.Dev.FunctionalParadigm; using AStar.Dev.Wallpaper.Scrapper.Classifications; using AStar.Dev.Wallpaper.Scrapper.Dialogs; +using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.ScrapeConfigurationEditor; using AStar.Dev.Wallpaper.Scrapper.Services; using AStar.Dev.Wallpaper.Scrapper.Support; @@ -146,8 +147,9 @@ private Task> RunScrapeWorkflowAsync(CancellationToken ct) logger.Information("Configuring Playwright..."); logger.Information("Starting scrape..."); - return searchWorkflow.RunAsync(logger, ct) - .TapAsync(_ => logger.Information("Scrape completed...")); + return searchWorkflow.RunAsync(ct) + .TapAsync(_ => logger.Information("Scrape completed...")) + .ToStringError(); } private void ResetUI() diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Models/ImagePageOutcome.cs b/src/AStar.Dev.Wallpaper.Scrapper/Models/ImagePageOutcome.cs new file mode 100644 index 0000000..e8873ea --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Models/ImagePageOutcome.cs @@ -0,0 +1,19 @@ +using AStar.Dev.Wallpaper.Scrapper.Repositories; + +namespace AStar.Dev.Wallpaper.Scrapper.Models; + +/// The outcome of scraping an image page, replacing the historical ImagePageResult. +public abstract record ImagePageOutcome; + +/// The image page's tags were accepted; the image should be downloaded and saved. +/// The URL of the image to download. +/// The directory segments the image should be saved under. +/// The prefix to apply to the saved file name. +/// The tag text scraped from the image page. +/// The raw tag data scraped from the image page. +public sealed record ScrapedImage(string ImageUrl, IReadOnlyList DirectorySegments, string FilePrefix, IReadOnlyList Tags, IReadOnlyList RawTags) : ImagePageOutcome; + +/// The image page's tags matched an ignore rule; the image should not be downloaded. +/// The tag text scraped from the image page before the skip was detected. +/// The raw tag data scraped from the image page before the skip was detected. +public sealed record SkippedImage(IReadOnlyList Tags, IReadOnlyList RawTags) : ImagePageOutcome; diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Models/ImagePageOutcomeFactory.cs b/src/AStar.Dev.Wallpaper.Scrapper/Models/ImagePageOutcomeFactory.cs new file mode 100644 index 0000000..425096b --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Models/ImagePageOutcomeFactory.cs @@ -0,0 +1,29 @@ +using AStar.Dev.Guard.Clauses; +using AStar.Dev.Wallpaper.Scrapper.Repositories; + +namespace AStar.Dev.Wallpaper.Scrapper.Models; + +/// Factory methods for creating instances of the discriminated union. +public static class ImagePageOutcomeFactory +{ + /// Creates a outcome. + public static ScrapedImage CreateScrapedImage(string imageUrl, IReadOnlyList directorySegments, string filePrefix, IReadOnlyList tags, IReadOnlyList rawTags) + { + GuardAgainst.Null(imageUrl); + GuardAgainst.Null(directorySegments); + GuardAgainst.Null(filePrefix); + GuardAgainst.Null(tags); + GuardAgainst.Null(rawTags); + + return new(imageUrl, directorySegments, filePrefix, tags, rawTags); + } + + /// Creates a outcome. + public static SkippedImage CreateSkippedImage(IReadOnlyList tags, IReadOnlyList rawTags) + { + GuardAgainst.Null(tags); + GuardAgainst.Null(rawTags); + + return new(tags, rawTags); + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Models/ImagePageResult.cs b/src/AStar.Dev.Wallpaper.Scrapper/Models/ImagePageResult.cs deleted file mode 100644 index 90f7d11..0000000 --- a/src/AStar.Dev.Wallpaper.Scrapper/Models/ImagePageResult.cs +++ /dev/null @@ -1,3 +0,0 @@ -namespace AStar.Dev.Wallpaper.Scrapper.Models; - -public record ImagePageResult(string? ImageUrl, List DirectoryName, string FilePrefix, bool Skip, IReadOnlyList Tags); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeError.cs b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeError.cs index 1fe4332..2a204e8 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeError.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeError.cs @@ -1,3 +1,5 @@ +using AStar.Dev.FunctionalParadigm; + namespace AStar.Dev.Wallpaper.Scrapper.Models; /// Base type for every error that a scrape pipeline operation can fail with. @@ -36,3 +38,18 @@ public sealed record ClassificationFailed(string FileName, string Message) : Scr /// An unanticipated exception was raised while running the scrape pipeline. /// The exception that was raised. public sealed record UnexpectedError(Exception Exception) : ScrapeError(Exception.Message); + +/// The dimensions of a downloaded image could not be read. +/// The path of the image whose dimensions failed to read. +/// A human-readable description of the failure. +public sealed record ImageDimensionReadFailed(string Path, string Message) : ScrapeError(Message); + +/// An export file could not be imported. +/// The path of the file that failed to import. +/// A human-readable description of the failure. +public sealed record ImportFailed(string FilePath, string Message) : ScrapeError(Message); + +/// An imported payload failed validation. +/// The accumulated validation errors. +/// A human-readable description of the failure. +public sealed record ValidationFailed(IReadOnlyList Errors, string Message) : ScrapeError(Message); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorFactory.cs b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorFactory.cs index 8191d23..f0e4488 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorFactory.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorFactory.cs @@ -1,3 +1,4 @@ +using AStar.Dev.FunctionalParadigm; using AStar.Dev.Guard.Clauses; namespace AStar.Dev.Wallpaper.Scrapper.Models; @@ -64,4 +65,31 @@ public static UnexpectedError CreateUnexpectedError(Exception exception) return new(exception); } + + /// Creates an error. + public static ImageDimensionReadFailed CreateImageDimensionReadFailed(string path, string message) + { + GuardAgainst.Null(path); + GuardAgainst.Null(message); + + return new(path, message); + } + + /// Creates an error. + public static ImportFailed CreateImportFailed(string filePath, string message) + { + GuardAgainst.Null(filePath); + GuardAgainst.Null(message); + + return new(filePath, message); + } + + /// Creates a error. + public static ValidationFailed CreateValidationFailed(IReadOnlyList errors, string message) + { + GuardAgainst.Null(errors); + GuardAgainst.Null(message); + + return new(errors, message); + } } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorLoggingExtensions.cs b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorLoggingExtensions.cs index fd49d78..f865087 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorLoggingExtensions.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorLoggingExtensions.cs @@ -8,5 +8,18 @@ public static class ScrapeErrorLoggingExtensions { /// Logs an error when is a failure, then returns unchanged. public static Result LogFailure(this Result result, ILogger logger) - => result.Tap(_ => { }, error => logger.Error("Scrape pipeline failure: {Message}", error.Message)); + { + ArgumentNullException.ThrowIfNull(result); + ArgumentNullException.ThrowIfNull(logger); + + return result.Tap(_ => { }, error => logger.Error("Scrape pipeline failure: {Message}", error.Message)); + } + + /// Logs an error when the awaited result is a failure, then returns the result unchanged. + public static Task> LogFailure(this Task> resultTask, ILogger logger) + { + ArgumentNullException.ThrowIfNull(resultTask); + + return resultTask.TapAsync(_ => { }, error => logger.Error("Scrape pipeline failure: {Message}", error.Message)); + } } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorResultExtensions.cs b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorResultExtensions.cs new file mode 100644 index 0000000..2f6691e --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Models/ScrapeErrorResultExtensions.cs @@ -0,0 +1,26 @@ +using AStar.Dev.FunctionalParadigm; + +namespace AStar.Dev.Wallpaper.Scrapper.Models; + +/// +/// Bridges pipelines into call sites that have not yet migrated off +/// string-typed errors, by mapping the down to its . +/// +public static class ScrapeErrorResultExtensions +{ + /// Maps a failure down to its message, preserving a success value unchanged. + public static Result ToStringError(this Result result) + { + ArgumentNullException.ThrowIfNull(result); + + return result.Match(value => Result.Success(value), error => Result.Failure(error.Message)); + } + + /// Maps a failure down to its message, preserving a success value unchanged. + public static async Task> ToStringError(this Task> resultTask) + { + ArgumentNullException.ThrowIfNull(resultTask); + + return (await resultTask.ConfigureAwait(false)).ToStringError(); + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Pages/ITopWallpapersPage.cs b/src/AStar.Dev.Wallpaper.Scrapper/Pages/ITopWallpapersPage.cs index 234ca1a..02825e7 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Pages/ITopWallpapersPage.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Pages/ITopWallpapersPage.cs @@ -1,12 +1,13 @@ -using Microsoft.Playwright; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; namespace AStar.Dev.Wallpaper.Scrapper.Pages; public interface ITopWallpapersPage { - Task LoadTopWallpapersPageAsync(int pageNumber); + Task> LoadTopWallpapersPageAsync(int pageNumber); - Task PageInfoAsync(); + Task> PageInfoAsync(); - Task> GetImagePageLinksAsync(); + Task, ScrapeError>> GetImagePageLinksAsync(); } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Pages/ImagePage.cs b/src/AStar.Dev.Wallpaper.Scrapper/Pages/ImagePage.cs index 3d9ea0a..d04cc18 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Pages/ImagePage.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Pages/ImagePage.cs @@ -1,3 +1,4 @@ +using AStar.Dev.FunctionalParadigm; using AStar.Dev.Utilities; using AStar.Dev.Wallpaper.Scrapper.DTOs; using AStar.Dev.Wallpaper.Scrapper.Models; @@ -8,33 +9,35 @@ namespace AStar.Dev.Wallpaper.Scrapper.Pages; -public sealed class ImagePage(IPlaywrightService playwrightService, ScrapeConfiguration scrapeConfiguration, TagsToIgnoreCompletely tagsToIgnoreCompletely, TagsTextToIgnore tagsTextToIgnore, IScrapedTagRepository scrapedTagRepository) +public sealed class ImagePage(IPlaywrightService playwrightService, ScrapeConfiguration scrapeConfiguration, TagsToIgnoreCompletely tagsToIgnoreCompletely, TagsTextToIgnore tagsTextToIgnore) { private IPage page = null!; - public async Task GetImageFromPageAsync(string link, string categoryName) - { - page ??= await playwrightService.ConfigurePlaywrightAsync(); - _ = await page.GotoAsync(link); + public async Task> GetImageFromPageAsync(string link, string categoryName) + => (await Try.RunAsync(() => LoadOutcomeAsync(link, categoryName)).ConfigureAwait(false)) + .ToResult(exception => ScrapeErrorFactory.CreatePageLoadFailed(link, exception.Message)); - var tagLocators = await page.Locator(".tagname").AllAsync(); - var tagData = await Task.WhenAll(tagLocators.Select(GetTagsAsync)); + private async Task LoadOutcomeAsync(string link, string categoryName) + { + await EnsurePageAsync().ConfigureAwait(false); + _ = await page.GotoAsync(link).ConfigureAwait(false); - await scrapedTagRepository.SaveAsync([.. tagData.Where(t => !string.IsNullOrWhiteSpace(t.Category))]); + var tagLocators = await page.Locator(".tagname").AllAsync().ConfigureAwait(false); + var tagData = await Task.WhenAll(tagLocators.Select(GetTagsAsync)).ConfigureAwait(false); string initialDirectory = scrapeConfiguration.ScrapeDirectories.BaseSaveDirectory.CombinePath(categoryName.Replace(' ', '-')); var context = TagRuleContextFactory.Create(initialDirectory, scrapeConfiguration.ScrapeDirectories.BaseDirectoryFamous, tagsToIgnoreCompletely, tagsTextToIgnore); var outcome = TagRules.Evaluate(tagData, context); - return await MapOutcomeToResultAsync(outcome); + return await MapOutcomeAsync(outcome, tagData).ConfigureAwait(false); } - private async Task MapOutcomeToResultAsync(TagOutcome outcome) + private async Task MapOutcomeAsync(TagOutcome outcome, IReadOnlyList rawTags) => outcome switch { - SkipImage skip => new ImagePageResult(null, [], string.Empty, true, skip.Tags), - Accept accept => new ImagePageResult(await GetImageSourceAsync(), [.. accept.DirectorySegments], accept.FilePrefix, false, accept.Tags), + SkipImage skip => ImagePageOutcomeFactory.CreateSkippedImage(skip.Tags, rawTags), + Accept accept => ImagePageOutcomeFactory.CreateScrapedImage(await GetImageSourceAsync().ConfigureAwait(false) ?? string.Empty, accept.DirectorySegments, accept.FilePrefix, accept.Tags, rawTags), _ => throw new InvalidOperationException("Unexpected tag outcome."), }; @@ -42,13 +45,16 @@ private async Task MapOutcomeToResultAsync(TagOutcome outcome) { var imageTag = page.Locator("#wallpaper"); - return await imageTag.GetAttributeAsync("src"); + return await imageTag.GetAttributeAsync("src").ConfigureAwait(false); } + private async Task EnsurePageAsync() => page ??= await playwrightService.ConfigurePlaywrightAsync().ConfigureAwait(false); + private static async Task GetTagsAsync(ILocator tag) { - string textTask = await tag.InnerTextAsync(); - string? attrTask = await tag.GetAttributeAsync("original-title"); + string textTask = await tag.InnerTextAsync().ConfigureAwait(false); + string? attrTask = await tag.GetAttributeAsync("original-title").ConfigureAwait(false); + return new TagData(textTask, attrTask); } } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Pages/SearchResultsPage.cs b/src/AStar.Dev.Wallpaper.Scrapper/Pages/SearchResultsPage.cs index 5a07089..e279208 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Pages/SearchResultsPage.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Pages/SearchResultsPage.cs @@ -1,12 +1,15 @@ using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Services; using Microsoft.Playwright; -using Serilog.Core; namespace AStar.Dev.Wallpaper.Scrapper.Pages; -public sealed class SearchResultsPage(IPlaywrightService playwrightService, Logger logger) +public sealed class SearchResultsPage(IPlaywrightService playwrightService) { + private const string PageHeaderErrorSource = "search-results-page-header"; + private const string ImagePageLinksErrorSource = "search-results-image-page-links"; + private IPage page = null!; private ILocator NewSubscriptionWallpapersHeader => page.GetByText("New Subscription Wallpapers", new PageGetByTextOptions { Exact = false, }); @@ -17,60 +20,44 @@ public sealed class SearchResultsPage(IPlaywrightService playwrightService, Logg private ILocator ImagePreviews => page.GetByRole(AriaRole.Link); - public async Task LoadSearchPageAsync(string searchString, int pageNumber) - { - try - { - page ??= await playwrightService.ConfigurePlaywrightAsync(); - return await GotoSearchPageAsync(searchString, pageNumber); - } - catch (Exception exception) - { - logger.Error(exception.GetBaseException().Message); + public async Task> LoadSearchPageAsync(string searchString, int pageNumber) + => (await Try.RunAsync(() => AttemptGotoSearchPageAsync(searchString, pageNumber)).ConfigureAwait(false)) + .ToResult(exception => ScrapeErrorFactory.CreatePageLoadFailed($"{searchString}{pageNumber}", exception.Message)) + .Bind(succeeded => ToSearchPageResult(succeeded, searchString, pageNumber)); - throw; - } - } + public async Task> PageInfoAsync() + => (await Try.RunAsync(GetPageHeaderAsync).ConfigureAwait(false)) + .ToResult(exception => ScrapeErrorFactory.CreatePageLoadFailed(PageHeaderErrorSource, exception.Message)) + .Bind(PageHeaderParser.Parse); - public async Task<(int pageCount, int imageCount, string subDirectoryName)> PageInfoAsync() + public async Task, ScrapeError>> ImagePageLinksAsync() + => (await Try.RunAsync(GetTheImagePageLinksAsync).ConfigureAwait(false)) + .ToResult, ScrapeError>(exception => ScrapeErrorFactory.CreatePageLoadFailed(ImagePageLinksErrorSource, exception.Message)); + + private static Result ToSearchPageResult(bool succeeded, string searchString, int pageNumber) { - try - { - page ??= await playwrightService.ConfigurePlaywrightAsync(); - return await GetPageInfoAsync(); - } - catch (Exception exception) - { - logger.Error(exception.GetBaseException().Message); + if (succeeded) return Unit.Value; - throw; - } + return ScrapeErrorFactory.CreatePageLoadFailed($"{searchString}{pageNumber}", $"Could not load the search results page for {searchString}{pageNumber} after retry."); } - public async Task> ImagePageLinksAsync() + private async Task AttemptGotoSearchPageAsync(string searchString, int pageNumber) { - try - { - page ??= await playwrightService.ConfigurePlaywrightAsync(); - return await GetTheImagePageLinksAsync(); - } - catch (Exception exception) - { - logger.Error(exception.GetBaseException().Message); + await EnsurePageAsync().ConfigureAwait(false); - throw; - } - } + var firstAttempt = await GotoPageAsync(searchString, pageNumber).ConfigureAwait(false); - private async Task GotoSearchPageAsync(string searchString, int pageNumber) - { - var searchPage = await GotoPageAsync(searchString, pageNumber); + if (firstAttempt is { Ok: true, }) return true; - return searchPage is { Ok: true, } ? searchPage : await GotoPageAsync(searchString, pageNumber); + var secondAttempt = await GotoPageAsync(searchString, pageNumber).ConfigureAwait(false); + + return secondAttempt is { Ok: true, }; } private async Task> GetTheImagePageLinksAsync() { + await EnsurePageAsync().ConfigureAwait(false); + var imagePreviews = await ImagePreviews.AllAsync().ConfigureAwait(false); List hrefs = []; foreach (var imagePreview in imagePreviews) hrefs.Add(await imagePreview.GetAttributeAsync("href").ConfigureAwait(false)); @@ -78,35 +65,28 @@ private async Task> GetTheImagePageLinksAsync() return ImageLinkSelector.SelectWanted(hrefs); } - private async Task<(int pageCount, int imageCount, string subDirectoryName)> GetPageInfoAsync() - { - string? text = await GetPageHeaderAsync(); - - if (text is null) return (0, 0, string.Empty); - - return PageHeaderParser.Parse(text).Match( - pageInfo => (pageInfo.PageCount, pageInfo.ImageCount, pageInfo.SubDirectoryName), - error => throw new InvalidOperationException(error.Message)); - } - private async Task GotoPageAsync(string searchString, int pageNumber) - => await page.GotoAsync($"{searchString}{pageNumber}", new PageGotoOptions { Timeout = 60000, }); + => await page.GotoAsync($"{searchString}{pageNumber}", new PageGotoOptions { Timeout = 60000, }).ConfigureAwait(false); private async Task GetPageHeaderAsync() { + await EnsurePageAsync().ConfigureAwait(false); + string? text; try { - text = await WallpaperSearchHeader.TextContentAsync(); + text = await WallpaperSearchHeader.TextContentAsync().ConfigureAwait(false); } catch { - text = await WallpaperSearchHeaderGeneral.TextContentAsync(); + text = await WallpaperSearchHeaderGeneral.TextContentAsync().ConfigureAwait(false); } - if (text?.Length == 0) text = await NewSubscriptionWallpapersHeader.TextContentAsync(); + if (text?.Length == 0) text = await NewSubscriptionWallpapersHeader.TextContentAsync().ConfigureAwait(false); return text; } + + private async Task EnsurePageAsync() => page ??= await playwrightService.ConfigurePlaywrightAsync().ConfigureAwait(false); } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Pages/SubscriptionsImagesListPage.cs b/src/AStar.Dev.Wallpaper.Scrapper/Pages/SubscriptionsImagesListPage.cs index 7ab74fe..7bee89c 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Pages/SubscriptionsImagesListPage.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Pages/SubscriptionsImagesListPage.cs @@ -7,33 +7,52 @@ namespace AStar.Dev.Wallpaper.Scrapper.Pages; public sealed class SubscriptionsImagesListPage(IPlaywrightService playwrightService, SearchConfiguration searchConfiguration) { + private const string PageHeaderErrorSource = "subscriptions-page-header"; + private const string ImagePageLinksErrorSource = "subscriptions-image-page-links"; + private const string ClearAllErrorSource = "subscriptions-clear-all"; + private IPage page = null!; private ILocator ImagePreviews => page.GetByRole(AriaRole.Link); private ILocator NewSubscriptionWallpapersHeader => page.GetByText("New Subscription Wallpapers", new PageGetByTextOptions { Exact = false, }); - public async Task LoadSubscriptionResultsPageAsync(int pageNumber) + public async Task> LoadSubscriptionResultsPageAsync(int pageNumber) + => (await Try.RunAsync(() => GotoSubscriptionsPageAsync(pageNumber)).ConfigureAwait(false)) + .ToResult(exception => ScrapeErrorFactory.CreatePageLoadFailed($"{searchConfiguration.Subscriptions}{pageNumber}", exception.Message)); + + public async Task> PageInfoAsync() + => (await Try.RunAsync(GetPageHeaderAsync).ConfigureAwait(false)) + .ToResult(exception => ScrapeErrorFactory.CreatePageLoadFailed(PageHeaderErrorSource, exception.Message)) + .Bind(SubscriptionHeaderParser.Parse); + + public async Task, ScrapeError>> GetImagePageLinksAsync() + => (await Try.RunAsync(GetTheImagePageLinksAsync).ConfigureAwait(false)) + .ToResult, ScrapeError>(exception => ScrapeErrorFactory.CreatePageLoadFailed(ImagePageLinksErrorSource, exception.Message)); + + public async Task> ClearAsync() + => (await Try.RunAsync(ClickClearAllSubscriptionsAsync).ConfigureAwait(false)) + .ToResult(exception => ScrapeErrorFactory.CreatePageLoadFailed(ClearAllErrorSource, exception.Message)); + + private async Task GotoSubscriptionsPageAsync(int pageNumber) { - page ??= await playwrightService.ConfigurePlaywrightAsync(); - return await page.GotoAsync($"{searchConfiguration.Subscriptions}{pageNumber}"); + await EnsurePageAsync().ConfigureAwait(false); + _ = await page.GotoAsync($"{searchConfiguration.Subscriptions}{pageNumber}").ConfigureAwait(false); + + return Unit.Value; } - public async Task<(int pageCount, string subDirectoryName)> PageInfoAsync() + private async Task GetPageHeaderAsync() { - page ??= await playwrightService.ConfigurePlaywrightAsync(); - string? text = await NewSubscriptionWallpapersHeader.TextContentAsync(); - - if (text is null) return (0, string.Empty); + await EnsurePageAsync().ConfigureAwait(false); - return SubscriptionHeaderParser.Parse(text).Match( - pageInfo => (pageInfo.PageCount, pageInfo.SubDirectoryName), - error => throw new InvalidOperationException(error.Message)); + return await NewSubscriptionWallpapersHeader.TextContentAsync().ConfigureAwait(false); } - public async Task> GetImagePageLinksAsync() + private async Task> GetTheImagePageLinksAsync() { - page ??= await playwrightService.ConfigurePlaywrightAsync(); + await EnsurePageAsync().ConfigureAwait(false); + var imagePreviews = await ImagePreviews.AllAsync().ConfigureAwait(false); List hrefs = []; foreach (var imagePreview in imagePreviews) hrefs.Add(await imagePreview.GetAttributeAsync("href").ConfigureAwait(false)); @@ -41,9 +60,17 @@ public async Task> GetImagePageLinksAsync() return ImageLinkSelector.SelectWanted(hrefs); } - public async Task ClearAsync() - => await page.Locator("div") - .Filter(new LocatorFilterOptions { HasText = " Clear All Subscriptions", }) - .GetByRole(AriaRole.Link, new LocatorGetByRoleOptions { Name = " Clear All Subscriptions", }) - .ClickAsync(); + private async Task ClickClearAllSubscriptionsAsync() + { + await EnsurePageAsync().ConfigureAwait(false); + + await page.Locator("div") + .Filter(new LocatorFilterOptions { HasText = " Clear All Subscriptions", }) + .GetByRole(AriaRole.Link, new LocatorGetByRoleOptions { Name = " Clear All Subscriptions", }) + .ClickAsync().ConfigureAwait(false); + + return Unit.Value; + } + + private async Task EnsurePageAsync() => page ??= await playwrightService.ConfigurePlaywrightAsync().ConfigureAwait(false); } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Pages/TopWallpapersPage.cs b/src/AStar.Dev.Wallpaper.Scrapper/Pages/TopWallpapersPage.cs index 7393ee8..4635a48 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Pages/TopWallpapersPage.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Pages/TopWallpapersPage.cs @@ -7,37 +7,53 @@ namespace AStar.Dev.Wallpaper.Scrapper.Pages; public sealed class TopWallpapersPage(IPlaywrightService playwrightService, SearchConfiguration searchConfiguration) : ITopWallpapersPage { + private const string PageHeaderErrorSource = "top-wallpapers-page-header"; + private const string ImagePageLinksErrorSource = "top-wallpapers-image-page-links"; + private IPage page = null!; private ILocator PageCount => page.GetByText("Page ", new PageGetByTextOptions { Exact = false, }); private ILocator ImagePreviews => page.GetByRole(AriaRole.Link); - public async Task LoadTopWallpapersPageAsync(int pageNumber) + public async Task> LoadTopWallpapersPageAsync(int pageNumber) + => (await Try.RunAsync(() => GotoTopWallpapersPageAsync(pageNumber)).ConfigureAwait(false)) + .ToResult(exception => ScrapeErrorFactory.CreatePageLoadFailed($"{searchConfiguration.TopWallpapers}{pageNumber}", exception.Message)); + + public async Task> PageInfoAsync() + => (await Try.RunAsync(GetPageHeaderAsync).ConfigureAwait(false)) + .ToResult(exception => ScrapeErrorFactory.CreatePageLoadFailed(PageHeaderErrorSource, exception.Message)) + .Bind(TopWallpapersHeaderParser.Parse); + + public async Task, ScrapeError>> GetImagePageLinksAsync() + => (await Try.RunAsync(GetTheImagePageLinksAsync).ConfigureAwait(false)) + .ToResult, ScrapeError>(exception => ScrapeErrorFactory.CreatePageLoadFailed(ImagePageLinksErrorSource, exception.Message)); + + private async Task GotoTopWallpapersPageAsync(int pageNumber) { - page ??= await playwrightService.ConfigurePlaywrightAsync(); - return _ = await page.GotoAsync($"{searchConfiguration.TopWallpapers}{pageNumber}"); + await EnsurePageAsync().ConfigureAwait(false); + _ = await page.GotoAsync($"{searchConfiguration.TopWallpapers}{pageNumber}").ConfigureAwait(false); + + return Unit.Value; } - public async Task PageInfoAsync() + private async Task GetPageHeaderAsync() { - page ??= await playwrightService.ConfigurePlaywrightAsync(); - string? text = await PageCount.First.TextContentAsync(); - - if (text is null) return 0; + await EnsurePageAsync().ConfigureAwait(false); - return TopWallpapersHeaderParser.Parse(text).Match( - pageCount => pageCount, - error => throw new InvalidOperationException(error.Message)); + return await PageCount.First.TextContentAsync().ConfigureAwait(false); } - public async Task> GetImagePageLinksAsync() + private async Task> GetTheImagePageLinksAsync() { - page ??= await playwrightService.ConfigurePlaywrightAsync(); + await EnsurePageAsync().ConfigureAwait(false); + var imagePreviews = await ImagePreviews.AllAsync().ConfigureAwait(false); List hrefs = []; foreach (var imagePreview in imagePreviews) hrefs.Add(await imagePreview.GetAttributeAsync("href").ConfigureAwait(false)); return ImageLinkSelector.SelectWanted(hrefs); } + + private async Task EnsurePageAsync() => page ??= await playwrightService.ConfigurePlaywrightAsync().ConfigureAwait(false); } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/ScrapeConfiguration/ScrapeConfigurationView.axaml.cs b/src/AStar.Dev.Wallpaper.Scrapper/ScrapeConfiguration/ScrapeConfigurationView.axaml.cs index df4df96..f63c031 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/ScrapeConfiguration/ScrapeConfigurationView.axaml.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/ScrapeConfiguration/ScrapeConfigurationView.axaml.cs @@ -1,4 +1,5 @@ using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Services; using Avalonia.Controls; using Avalonia.Interactivity; @@ -68,7 +69,7 @@ private async void OnImportScrapeConfigClicked(object? sender, RoutedEventArgs e } ) .Tap(_ => { logger.Information("Importing scrape configuration..."); ViewModel.UpdateStatus("Importing scrape configuration..."); }) - .Bind(_ => importExportService.ImportScrapeConfigurationFromFile()) + .Bind(_ => importExportService.ImportScrapeConfigurationFromFile().ToStringError()) .MapAsync(entity => scrapeConfigurationService.ImportScrapeConfigurationAsync(entity, cts!.Token)) .TapAsync(_ => { logger.Information("Scrape configuration import completed..."); ViewModel.UpdateStatus("Import completed."); }) .EnsureAsync(() => ResetUI()); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Services/ClassificationMatcher.cs b/src/AStar.Dev.Wallpaper.Scrapper/Services/ClassificationMatcher.cs new file mode 100644 index 0000000..6547e67 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Services/ClassificationMatcher.cs @@ -0,0 +1,27 @@ +using AStar.Dev.Guard.Clauses; +using AStar.Dev.Infrastructure.AppDb.Entities; +using AStar.Dev.Wallpaper.Scrapper.Support; + +namespace AStar.Dev.Wallpaper.Scrapper.Services; + +/// +/// Pure classification matching, frozen from the historical FileClassificationService.CollectFileNameMatches +/// and category-classification steps. Tag matches are excluded because resolving them requires a database lookup. +/// +public static class ClassificationMatcher +{ + /// Matches 's searchable classifications and category classification against 's name. + public static IReadOnlyList Match(PageClassificationData pageData, FileDetailEntity fileDetail) + { + GuardAgainst.Null(pageData); + GuardAgainst.Null(fileDetail); + + var fileNameMatches = pageData.SearchableClassifications + .Where(entry => entry.Keywords.Any(keyword => fileDetail.FullNameWithPath().Contains(keyword, StringComparison.OrdinalIgnoreCase))) + .Select(entry => entry.Category); + + return pageData.CategoryClassification is null + ? [.. fileNameMatches,] + : [.. fileNameMatches, pageData.CategoryClassification,]; + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Services/FileClassificationService.cs b/src/AStar.Dev.Wallpaper.Scrapper/Services/FileClassificationService.cs index a66dae2..7384276 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Services/FileClassificationService.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Services/FileClassificationService.cs @@ -2,6 +2,7 @@ using AStar.Dev.Infrastructure.AppDb; using AStar.Dev.Infrastructure.AppDb.Entities; using AStar.Dev.Utilities; +using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Support; using Microsoft.EntityFrameworkCore; @@ -38,33 +39,9 @@ public async Task LoadPageClassificationDataAsync(string return PageClassificationDataFactory.Create(searchable, categoryClassification, includedTags); } - public async Task ClassifyAsync(FileDetailEntity fileDetail, PageClassificationData pageData, IReadOnlyList imageTags, CancellationToken token) - { - await using var context = await contextFactory.CreateDbContextAsync(token).ConfigureAwait(false); - - if (await context.FileClassifications.AnyAsync(classification => classification.FileDetailId == fileDetail.Id, token).ConfigureAwait(false)) - return; - - var matched = new List(); - - CollectFileNameMatches(pageData.SearchableClassifications, fileDetail, matched); - if (pageData.CategoryClassification is not null) - matched.Add(pageData.CategoryClassification); - await CollectTagMatchesAsync(context, pageData.IncludedTags, imageTags, matched, token).ConfigureAwait(false); - - var distinct = matched.DistinctBy(c => c.Name).ToList(); - - await context.SaveChangesAsync(token).ConfigureAwait(false); - - foreach (var classification in distinct) - context.FileClassifications.Add(new FileClassificationEntity - { - FileDetailId = fileDetail.Id, - CategoryId = classification.Id - }); - - await context.SaveChangesAsync(token).ConfigureAwait(false); - } + public async Task> ClassifyAsync(FileDetailEntity fileDetail, PageClassificationData pageData, IReadOnlyList imageTags, CancellationToken token) + => (await Try.RunAsync(() => ClassifyInternalAsync(fileDetail, pageData, imageTags, token)).ConfigureAwait(false)) + .ToResult(exception => ScrapeErrorFactory.CreateClassificationFailed(fileDetail.FileName.Value, exception.Message)); internal async Task<(List Categories, List Keywords)> ExportClassificationsAsync(CancellationToken token) { @@ -121,6 +98,37 @@ internal async Task ImportClassificationsAsync((List ClassifyInternalAsync(FileDetailEntity fileDetail, PageClassificationData pageData, IReadOnlyList imageTags, CancellationToken token) + { + await using var context = await contextFactory.CreateDbContextAsync(token).ConfigureAwait(false); + + if (await context.FileClassifications.AnyAsync(classification => classification.FileDetailId == fileDetail.Id, token).ConfigureAwait(false)) + return Unit.Value; + + var matched = new List(ClassificationMatcher.Match(pageData, fileDetail)); + await CollectTagMatchesAsync(context, pageData.IncludedTags, imageTags, matched, token).ConfigureAwait(false); + + var distinct = matched.DistinctBy(c => c.Name).ToList(); + + foreach (var classification in distinct) + { + EnsureTracked(context, classification); + context.FileClassifications.Add(new FileClassificationEntity { FileDetailId = fileDetail.Id, Category = classification }); + } + + await context.SaveChangesAsync(token).ConfigureAwait(false); + + return Unit.Value; + } + + private static void EnsureTracked(AppDbContext context, FileClassificationCategoryEntity classification) + { + var entry = context.Entry(classification); + + if (entry.State == EntityState.Detached && classification.Id != 0) + entry.State = EntityState.Unchanged; + } + private static async Task ResolveCategoryClassificationAsync(AppDbContext context, string categoryId, CancellationToken token) { if (string.IsNullOrEmpty(categoryId)) return null; @@ -142,11 +150,6 @@ internal async Task ImportClassificationsAsync((List Keywords)> searchable, FileDetailEntity fileDetail, List matched) - => matched.AddRange(searchable - .Where(entry => entry.Keywords.Any(keyword => fileDetail.FullNameWithPath().Contains(keyword, StringComparison.OrdinalIgnoreCase))) - .Select(entry => entry.Category)); - private static async Task CollectTagMatchesAsync(AppDbContext context, IReadOnlyList includedTags, IReadOnlyList imageTags, List matched, CancellationToken token) { if (imageTags.Count == 0) return; diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Services/IImportExportService.cs b/src/AStar.Dev.Wallpaper.Scrapper/Services/IImportExportService.cs index 097a715..dc3c893 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Services/IImportExportService.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Services/IImportExportService.cs @@ -1,5 +1,6 @@ using AStar.Dev.FunctionalParadigm; using AStar.Dev.Infrastructure.AppDb.Entities; +using AStar.Dev.Wallpaper.Scrapper.Models; using FileClassificationDomain = AStar.Dev.Infrastructure.AppDb.Entities.FileClassificationCategoryEntity; using FileClassificationKeywordDomain = AStar.Dev.Infrastructure.AppDb.Entities.FileClassificationKeywordEntity; using ScrapedTagDomain = AStar.Dev.Infrastructure.AppDb.Entities.ScrapedTagEntity; @@ -9,10 +10,10 @@ namespace AStar.Dev.Wallpaper.Scrapper.Services; public interface IImportExportService { void ExportFileClassificationsToFile((List Categories, List Keywords) classifications); - Result<(List Categories, List Keywords), string> ImportFileClassificationsFromFile(); + Result<(List Categories, List Keywords), ScrapeError> ImportFileClassificationsFromFile(); void ExportScrapeConfigurationToFile(ScrapeConfigurationEntity entity); - Result ImportScrapeConfigurationFromFile(); + Result ImportScrapeConfigurationFromFile(); void ExportScrapedTagsToFile(List tags); - Result, string> ImportScrapedTagsFromFile(); + Result, ScrapeError> ImportScrapedTagsFromFile(); } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Services/ImagePageService.cs b/src/AStar.Dev.Wallpaper.Scrapper/Services/ImagePageService.cs index 20ac6c4..367b958 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Services/ImagePageService.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Services/ImagePageService.cs @@ -7,110 +7,149 @@ using AStar.Dev.Wallpaper.Scrapper.Repositories; using AStar.Dev.Wallpaper.Scrapper.Support; using Serilog.Core; -using SkiaSharp; namespace AStar.Dev.Wallpaper.Scrapper.Services; -public sealed class ImagePageService(ImagePage imagePage, IFileDetailRepository fileDetailRepository, FileClassificationService fileClassificationService, ScrapeConfiguration scrapeConfiguration, TimeProvider timeProvider, Logger logger, IDirectoryHelper directoryHelper, ImageBroadcaster imageBroadcaster, IDelayStrategy delayStrategy, IImageRetriever imageRetriever, IImageSaver imageSaver, IFileSystem fileSystem) +public sealed class ImagePageService( + ImagePage imagePage, + IFileDetailRepository fileDetailRepository, + FileClassificationService fileClassificationService, + ScrapeConfiguration scrapeConfiguration, + TimeProvider timeProvider, + Logger logger, + IDirectoryHelper directoryHelper, + ImageBroadcaster imageBroadcaster, + IDelayStrategy delayStrategy, + IImageRetriever imageRetriever, + IImageSaver imageSaver, + IFileSystem fileSystem, + IScrapedTagRepository scrapedTagRepository, + IImageDimensionReader imageDimensionReader) { private const int LoggedPathTailLength = 50; /// Retained for constructor-signature stability. The image-pause delay it previously drove now lives in , which is configured with its own . private readonly ScrapeConfiguration scrapeConfiguration = scrapeConfiguration; - public async Task GetTheImagePagesAsync(IReadOnlyCollection imagePageLinks, string categoryId, string name, CancellationToken ct = default) + /// Retained for constructor-signature stability. File size is now computed from the downloaded bytes directly, so a saved image's entry no longer needs to be re-read. + private readonly IFileSystem fileSystem = fileSystem; + + public async Task> GetTheImagePagesAsync(IReadOnlyCollection imagePageLinks, string categoryId, string name, CancellationToken ct = default) { var pageData = await fileClassificationService.LoadPageClassificationDataAsync(categoryId, ct).ConfigureAwait(false); foreach (string pageLink in imagePageLinks) { ct.ThrowIfCancellationRequested(); - try - { - string fileName = Path.GetFileName(pageLink); - - if (await fileDetailRepository.ExistsAsync(fileName).ConfigureAwait(false)) - { - logger.Information("Not downloading {fileName} as we already have it...{Timestamp:HH:mm:ss:fff} (UTC)", fileName, timeProvider.GetUtcNow()); - await delayStrategy.DelayAsync(DelayKind.ImageAlreadyDownloaded, ct).ConfigureAwait(false); - continue; - } + string fileName = Path.GetFileName(pageLink); - await ProcessImagePageAsync(pageLink, name, pageData, ct).ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException) + if (await fileDetailRepository.ExistsAsync(fileName).ConfigureAwait(false)) { - logger.Warning(ex, "Failed to process {pageLink}, retrying after delay.", pageLink); - await delayStrategy.DelayAsync(DelayKind.Retry, ct).ConfigureAwait(false); - await ProcessImagePageAsync(pageLink, name, pageData, ct).ConfigureAwait(false); + logger.Information("Not downloading {fileName} as we already have it...{Timestamp:HH:mm:ss:fff} (UTC)", fileName, timeProvider.GetUtcNow()); + await delayStrategy.DelayAsync(DelayKind.ImageAlreadyDownloaded, ct).ConfigureAwait(false); + continue; } + + var pageResult = await ProcessImagePageAsync(pageLink, name, pageData, ct).ConfigureAwait(false); + var pageFailed = pageResult.Match(_ => false, _ => true); + + if (pageFailed) return pageResult; } + + return Unit.Value; } - private async Task ProcessImagePageAsync(string pageLink, string name, PageClassificationData pageData, CancellationToken ct) + public async Task> ProcessImagePageAsync(string pageLink, string categoryName, PageClassificationData pageData, CancellationToken ct) { - try + await delayStrategy.DelayAsync(DelayKind.BeforeImage, ct).ConfigureAwait(false); + + return await imagePage.GetImageFromPageAsync(pageLink, categoryName) + .BindAsync(outcome => HandleOutcomeAsync(outcome, categoryName, pageData, ct)) + .ConfigureAwait(false); + } + + private async Task> HandleOutcomeAsync(ImagePageOutcome outcome, string categoryName, PageClassificationData pageData, CancellationToken ct) + { + await SaveScrapedTagsAsync(outcome).ConfigureAwait(false); + + return outcome switch { - await delayStrategy.DelayAsync(DelayKind.BeforeImage, ct).ConfigureAwait(false); + SkippedImage skipped => LogSkippedImage(categoryName, skipped), + ScrapedImage scraped => await DownloadAndPersistAsync(scraped, pageData, ct).ConfigureAwait(false), + _ => throw new InvalidOperationException("Unexpected image page outcome."), + }; + } - var result = await imagePage.GetImageFromPageAsync(pageLink, name).ConfigureAwait(false); - if (result.Skip || result.ImageUrl is null) - { - logger.Information("Skipping {Name} with Tags: {Tags}", name, string.Join(", ", result.Tags)); - return; - } + private Task SaveScrapedTagsAsync(ImagePageOutcome outcome) + { + var rawTags = outcome switch + { + ScrapedImage scraped => scraped.RawTags, + SkippedImage skipped => skipped.RawTags, + _ => throw new InvalidOperationException("Unexpected image page outcome."), + }; - var directoryName = directoryHelper.CreateDirectoryIfRequired(result.DirectoryName); + return scrapedTagRepository.SaveAsync([.. rawTags.Where(tag => !string.IsNullOrWhiteSpace(tag.Category)),]); + } + + private Result LogSkippedImage(string categoryName, SkippedImage skipped) + { + logger.Information("Skipping {Name} with Tags: {Tags}", categoryName, string.Join(", ", skipped.Tags)); - string filename = ScrapedFileNameFactory.Create(result.FilePrefix, result.ImageUrl); + return Unit.Value; + } - string imageNameWithPath = directoryName.Value.CombinePath(filename); - byte[] image = (await imageRetriever.GetImageAsync(result.ImageUrl, ct).ConfigureAwait(false)) - .Match(bytes => bytes, error => throw new InvalidOperationException(error.Message)); - logger.Information("About to save {filename} to ...{imageNameWithPath} as we don't appear to have it.", filename, TruncatedForLogging(imageNameWithPath)); - await imageSaver.SaveAsync(image, imageNameWithPath).ConfigureAwait(false); - imageBroadcaster.Broadcast(imageNameWithPath); + private async Task> DownloadAndPersistAsync(ScrapedImage scraped, PageClassificationData pageData, CancellationToken ct) + { + var directoryName = directoryHelper.CreateDirectoryIfRequired([.. scraped.DirectorySegments,]); + string filename = ScrapedFileNameFactory.Create(scraped.FilePrefix, scraped.ImageUrl); + string imageNameWithPath = directoryName.Value.CombinePath(filename); + + return await RetryExtensions.RetryOnceAsync( + () => imageRetriever.GetImageAsync(scraped.ImageUrl, ct), + () => delayStrategy.DelayAsync(DelayKind.Retry, ct)) + .BindAsync(image => SaveAndPersistAsync(image, imageNameWithPath, filename, directoryName, scraped, pageData, ct)) + .ConfigureAwait(false); + } - var fileInfo = fileSystem.FileInfo.New(imageNameWithPath); - var fileDetail = new FileDetailEntity - { - DirectoryName = directoryName, - FileName = new FileName(filename), - FileSize = fileInfo.Length, - IsImage = filename.IsImage() - }; + private async Task> SaveAndPersistAsync(byte[] image, string imageNameWithPath, string filename, DirectoryName directoryName, ScrapedImage scraped, PageClassificationData pageData, CancellationToken ct) + { + logger.Information("About to save {filename} to ...{imageNameWithPath} as we don't appear to have it.", filename, TruncatedForLogging(imageNameWithPath)); - if (fileDetail.IsImage) - { - var imageDetail = SKImage.FromEncodedData(imageNameWithPath); - if (imageDetail is not null) - { - fileDetail.Height = imageDetail.Height; - fileDetail.Width = imageDetail.Width; - fileDetail.ImageDetail = new ImageDetailEntity { Width = imageDetail.Width, Height = imageDetail.Height }; - } - } + return await imageSaver.SaveAsync(image, imageNameWithPath) + .TapAsync(_ => imageBroadcaster.Broadcast(imageNameWithPath)) + .BindAsync(_ => PersistFileDetailAsync(image, imageNameWithPath, filename, directoryName, scraped, pageData, ct)) + .ConfigureAwait(false); + } - await fileDetailRepository.AddAsync(fileDetail).ConfigureAwait(false); - await fileClassificationService.ClassifyAsync(fileDetail, pageData, result.Tags, ct).ConfigureAwait(false); - } - catch(TaskCanceledException) - { - logger.Warning("Task was cancelled while processing {pageLink}.", pageLink); - throw; - } - catch (OperationCanceledException) - { - logger.Warning("Operation was cancelled while processing {pageLink}.", pageLink); - throw; - } - catch (Exception ex) when (ex is not OperationCanceledException) + private async Task> PersistFileDetailAsync(byte[] image, string imageNameWithPath, string filename, DirectoryName directoryName, ScrapedImage scraped, PageClassificationData pageData, CancellationToken ct) + { + var fileDetail = new FileDetailEntity { - logger.Error(ex, "Error processing image page {pageLink}: {Message}", pageLink, ex.Message); - throw; - } + DirectoryName = directoryName, + FileName = new FileName(filename), + FileSize = image.Length, + IsImage = filename.IsImage() + }; + + ApplyImageDimensions(fileDetail, image, imageNameWithPath); + + await fileDetailRepository.AddAsync(fileDetail).ConfigureAwait(false); + + return await fileClassificationService.ClassifyAsync(fileDetail, pageData, scraped.Tags, ct).ConfigureAwait(false); } + private void ApplyImageDimensions(FileDetailEntity fileDetail, byte[] image, string imageNameWithPath) + => imageDimensionReader.Read(image, imageNameWithPath) + .Tap( + dimensions => + { + fileDetail.Width = dimensions.Width; + fileDetail.Height = dimensions.Height; + fileDetail.ImageDetail = new ImageDetailEntity { Width = dimensions.Width, Height = dimensions.Height }; + }, + error => logger.Warning("Could not read image dimensions for {imageNameWithPath}: {Message}", TruncatedForLogging(imageNameWithPath), error.Message)); + private static string TruncatedForLogging(string imageNameWithPath) => imageNameWithPath.Length > LoggedPathTailLength ? imageNameWithPath[^LoggedPathTailLength..] : imageNameWithPath; } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Services/ImportExportService.cs b/src/AStar.Dev.Wallpaper.Scrapper/Services/ImportExportService.cs index bccb358..0497859 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Services/ImportExportService.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Services/ImportExportService.cs @@ -3,6 +3,7 @@ using AStar.Dev.Infrastructure.AppDb.Entities; using AStar.Dev.Utilities; using AStar.Dev.Wallpaper.Scrapper.DTOs; +using AStar.Dev.Wallpaper.Scrapper.Models; using Serilog; using FileClassificationDomain = AStar.Dev.Infrastructure.AppDb.Entities.FileClassificationCategoryEntity; using FileClassificationDto = AStar.Dev.Wallpaper.Scrapper.DTOs.FileClassification; @@ -29,13 +30,13 @@ public void ExportFileClassificationsToFile((List Cate } } - public Result<(List Categories, List Keywords), string> ImportFileClassificationsFromFile() + public Result<(List Categories, List Keywords), ScrapeError> ImportFileClassificationsFromFile() { if (!fileSystem.File.Exists(ApplicationMetadata.FileClassificationsExportFilePath)) { logger.Error("File not found: {FilePath}", ApplicationMetadata.FileClassificationsExportFilePath); - return $"Error: File not found - {ApplicationMetadata.FileClassificationsExportFilePath}"; + return ScrapeErrorFactory.CreateImportFailed(ApplicationMetadata.FileClassificationsExportFilePath, $"Error: File not found - {ApplicationMetadata.FileClassificationsExportFilePath}"); } string classificationsJson = fileSystem.File.ReadAllText(ApplicationMetadata.FileClassificationsExportFilePath); @@ -45,10 +46,12 @@ public void ExportFileClassificationsToFile((List Cate { logger.Error("Failed to deserialize classifications from file: {FilePath}", ApplicationMetadata.FileClassificationsExportFilePath); - return $"Error: Failed to deserialize classifications from file - {ApplicationMetadata.FileClassificationsExportFilePath}"; + return ScrapeErrorFactory.CreateImportFailed(ApplicationMetadata.FileClassificationsExportFilePath, $"Error: Failed to deserialize classifications from file - {ApplicationMetadata.FileClassificationsExportFilePath}"); } - return classifications.ToDomain(); + return FileClassificationDtoValidator.ValidateAll(classifications) + .ToResult, ScrapeError>(errors => ScrapeErrorFactory.CreateValidationFailed(errors, "Classification import validation failed.")) + .Map(validDtos => validDtos.ToList().ToDomain()); } public void ExportScrapeConfigurationToFile(ScrapeConfigurationEntity entity) @@ -81,13 +84,13 @@ public void ExportScrapedTagsToFile(List tags) } } - public Result ImportScrapeConfigurationFromFile() + public Result ImportScrapeConfigurationFromFile() { if (!fileSystem.File.Exists(ApplicationMetadata.ScrapeConfigurationExportFilePath)) { logger.Error("File not found: {FilePath}", ApplicationMetadata.ScrapeConfigurationExportFilePath); - return $"Error: File not found - {ApplicationMetadata.ScrapeConfigurationExportFilePath}"; + return ScrapeErrorFactory.CreateImportFailed(ApplicationMetadata.ScrapeConfigurationExportFilePath, $"Error: File not found - {ApplicationMetadata.ScrapeConfigurationExportFilePath}"); } string json = fileSystem.File.ReadAllText(ApplicationMetadata.ScrapeConfigurationExportFilePath); @@ -97,19 +100,19 @@ public Result ImportScrapeConfigurationFromFi { logger.Error("Failed to deserialize scrape configuration from file: {FilePath}", ApplicationMetadata.ScrapeConfigurationExportFilePath); - return $"Error: Failed to deserialize scrape configuration from file - {ApplicationMetadata.ScrapeConfigurationExportFilePath}"; + return ScrapeErrorFactory.CreateImportFailed(ApplicationMetadata.ScrapeConfigurationExportFilePath, $"Error: Failed to deserialize scrape configuration from file - {ApplicationMetadata.ScrapeConfigurationExportFilePath}"); } return dto.ToDomain(); } - public Result, string> ImportScrapedTagsFromFile() + public Result, ScrapeError> ImportScrapedTagsFromFile() { if (!fileSystem.File.Exists(ApplicationMetadata.ScrapedTagsExportFilePath)) { logger.Error("File not found: {FilePath}", ApplicationMetadata.ScrapedTagsExportFilePath); - return $"Error: File not found - {ApplicationMetadata.ScrapedTagsExportFilePath}"; + return ScrapeErrorFactory.CreateImportFailed(ApplicationMetadata.ScrapedTagsExportFilePath, $"Error: File not found - {ApplicationMetadata.ScrapedTagsExportFilePath}"); } string tagsJson = fileSystem.File.ReadAllText(ApplicationMetadata.ScrapedTagsExportFilePath); @@ -119,7 +122,7 @@ public Result, string> ImportScrapedTagsFromFile() { logger.Error("Failed to deserialize tags from file: {FilePath}", ApplicationMetadata.ScrapedTagsExportFilePath); - return $"Error: Failed to deserialize tags from file - {ApplicationMetadata.ScrapedTagsExportFilePath}"; + return ScrapeErrorFactory.CreateImportFailed(ApplicationMetadata.ScrapedTagsExportFilePath, $"Error: Failed to deserialize tags from file - {ApplicationMetadata.ScrapedTagsExportFilePath}"); } return tags.ToDomain(timeProvider); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/CategoryMerger.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/CategoryMerger.cs new file mode 100644 index 0000000..21d54ab --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/CategoryMerger.cs @@ -0,0 +1,29 @@ +using AStar.Dev.Guard.Clauses; +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// Pure category-merge logic, frozen from the historical ConfigurationSaver.UpdateAndSaveTheConfigurationAsync upsert loop. +public static class CategoryMerger +{ + /// + /// Partitions (de-duplicated by , keeping the first + /// occurrence) into categories that match an id already in and categories that + /// do not. + /// + public static (IReadOnlyList ToUpdate, IReadOnlyList ToAdd) MergeCategories( + IEnumerable<(string Id, string Name, int LastKnownImageCount, int LastPageVisited, int TotalPages)> existing, + IReadOnlyList updated) + { + GuardAgainst.Null(existing); + GuardAgainst.Null(updated); + + var existingIds = existing.Select(category => category.Id).ToHashSet(); + var dedupedUpdated = updated.DistinctBy(category => category.Id).ToList(); + + IReadOnlyList toUpdate = [.. dedupedUpdated.Where(category => existingIds.Contains(category.Id)),]; + IReadOnlyList toAdd = [.. dedupedUpdated.Where(category => !existingIds.Contains(category.Id)),]; + + return (toUpdate, toAdd); + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/ConfigurationSaver.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/ConfigurationSaver.cs index d0a3128..7f6e905 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Support/ConfigurationSaver.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/ConfigurationSaver.cs @@ -1,3 +1,4 @@ +using AStar.Dev.FunctionalParadigm; using AStar.Dev.Infrastructure.AppDb; using AStar.Dev.Infrastructure.AppDb.Entities; using AStar.Dev.Wallpaper.Scrapper.Models; @@ -8,55 +9,54 @@ namespace AStar.Dev.Wallpaper.Scrapper.Support; public sealed class ConfigurationSaver(ScrapeConfiguration scrapeConfiguration, Logger logger, IDbContextFactory contextFactory) { - public async Task SaveUpdatedConfigurationAsync() - { - try - { - await UpdateAndSaveTheConfigurationAsync(); - } - catch (Exception exception) - { - logger.Error(exception.GetBaseException().Message); - throw; - } - } + public async Task> SaveUpdatedConfigurationAsync() + => (await Try.RunAsync(UpdateAndSaveTheConfigurationAsync).ConfigureAwait(false)) + .Tap(_ => { }, exception => logger.Error(exception.GetBaseException().Message)) + .ToResult(exception => ScrapeErrorFactory.CreateConfigurationSaveFailed(exception.Message)); - private async Task UpdateAndSaveTheConfigurationAsync() + private async Task UpdateAndSaveTheConfigurationAsync() { - await using var dbContext = await contextFactory.CreateDbContextAsync(); + await using var dbContext = await contextFactory.CreateDbContextAsync().ConfigureAwait(false); var entity = await dbContext.ScrapeConfiguration .Include(e => e.SearchConfiguration) .ThenInclude(sc => sc.SearchCategories) - .SingleAsync(); + .SingleAsync().ConfigureAwait(false); + + var existingCategories = entity.SearchConfiguration.SearchCategories + .Select(existing => (existing.Id, existing.Name, existing.LastKnownImageCount, existing.LastPageVisited, existing.TotalPages)); + + var (toUpdate, toAdd) = CategoryMerger.MergeCategories(existingCategories, scrapeConfiguration.SearchConfiguration.SearchCategories); + + ApplyUpdates(entity, toUpdate); + ApplyAdditions(entity, toAdd); - var dedupedCategories = scrapeConfiguration.SearchConfiguration.SearchCategories - .DistinctBy(c => c.Id) - .ToList(); + await dbContext.SaveChangesAsync().ConfigureAwait(false); - foreach (var cat in dedupedCategories) + return Unit.Value; + } + + private static void ApplyUpdates(ScrapeConfigurationEntity entity, IReadOnlyList toUpdate) + { + foreach (var category in toUpdate) { - var existing = entity.SearchConfiguration.SearchCategories - .FirstOrDefault(ec => ec.Id == cat.Id); - if (existing != null) - { - existing.LastKnownImageCount = cat.LastKnownImageCount; - existing.LastPageVisited = cat.LastPageVisited; - existing.TotalPages = cat.TotalPages; - } - else - { - entity.SearchConfiguration.SearchCategories.Add(new SearchCategoryEntity - { - SearchConfigurationId = entity.SearchConfiguration.Id, - Id = cat.Id, - Name = cat.Name, - LastKnownImageCount = cat.LastKnownImageCount, - LastPageVisited = cat.LastPageVisited, - TotalPages = cat.TotalPages, - }); - } + var existing = entity.SearchConfiguration.SearchCategories.First(ec => ec.Id == category.Id); + existing.LastKnownImageCount = category.LastKnownImageCount; + existing.LastPageVisited = category.LastPageVisited; + existing.TotalPages = category.TotalPages; } + } - await dbContext.SaveChangesAsync(); + private static void ApplyAdditions(ScrapeConfigurationEntity entity, IReadOnlyList toAdd) + { + foreach (var category in toAdd) + entity.SearchConfiguration.SearchCategories.Add(new SearchCategoryEntity + { + SearchConfigurationId = entity.SearchConfiguration.Id, + Id = category.Id, + Name = category.Name, + LastKnownImageCount = category.LastKnownImageCount, + LastPageVisited = category.LastPageVisited, + TotalPages = category.TotalPages, + }); } } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/IImageDimensionReader.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/IImageDimensionReader.cs new file mode 100644 index 0000000..bc59748 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/IImageDimensionReader.cs @@ -0,0 +1,11 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// Reads the pixel dimensions of a downloaded image's bytes. +public interface IImageDimensionReader +{ + /// Reads the dimensions of , previously saved to . + Result Read(byte[] image, string path); +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/IImageSaver.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/IImageSaver.cs index 09eb5ea..f13828c 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Support/IImageSaver.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/IImageSaver.cs @@ -1,8 +1,11 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; + namespace AStar.Dev.Wallpaper.Scrapper.Support; /// Saves a scraped image's bytes to disk. public interface IImageSaver { /// Saves to , cleaning the path first. - Task SaveAsync(byte[] image, string path); + Task> SaveAsync(byte[] image, string path); } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageDimensionReader.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageDimensionReader.cs new file mode 100644 index 0000000..1dbd6c6 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageDimensionReader.cs @@ -0,0 +1,25 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; +using SkiaSharp; + +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// The production , backed by , frozen from the historical ImagePageService dimension-probing behaviour. +public sealed class ImageDimensionReader : IImageDimensionReader +{ + /// + public Result Read(byte[] image, string path) + { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + + if (image.Length == 0) + return ScrapeErrorFactory.CreateImageDimensionReadFailed(path, "The image bytes were empty."); + + using var decoded = SKImage.FromEncodedData(image); + + return decoded is null + ? ScrapeErrorFactory.CreateImageDimensionReadFailed(path, "The image bytes could not be decoded.") + : ImageDimensionsFactory.Create(decoded.Width, decoded.Height); + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageDimensions.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageDimensions.cs new file mode 100644 index 0000000..5781566 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageDimensions.cs @@ -0,0 +1,6 @@ +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// The pixel dimensions of a decoded image. +/// The image width, in pixels. +/// The image height, in pixels. +public sealed record ImageDimensions(int Width, int Height); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageDimensionsFactory.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageDimensionsFactory.cs new file mode 100644 index 0000000..af8dcbf --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageDimensionsFactory.cs @@ -0,0 +1,14 @@ +namespace AStar.Dev.Wallpaper.Scrapper.Support; + +/// Factory methods for creating instances of . +public static class ImageDimensionsFactory +{ + /// Creates an for the given and . + public static ImageDimensions Create(int width, int height) + { + ArgumentOutOfRangeException.ThrowIfNegative(width); + ArgumentOutOfRangeException.ThrowIfNegative(height); + + return new(width, height); + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageSaver.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageSaver.cs index 2b5931c..3423be9 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageSaver.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/ImageSaver.cs @@ -1,5 +1,7 @@ using System.IO.Abstractions; +using AStar.Dev.FunctionalParadigm; using AStar.Dev.Utilities; +using AStar.Dev.Wallpaper.Scrapper.Models; namespace AStar.Dev.Wallpaper.Scrapper.Support; @@ -7,12 +9,23 @@ namespace AStar.Dev.Wallpaper.Scrapper.Support; public sealed class ImageSaver(IFileSystem fileSystem) : IImageSaver { /// - public async Task SaveAsync(byte[] image, string path) + public async Task> SaveAsync(byte[] image, string path) { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(path); + string cleanedPath = path.CleanPath(); if (cleanedPath.LastIndexOf(':') > 2) cleanedPath = cleanedPath[..2] + cleanedPath[2..].Replace(":", "_"); + return (await Try.RunAsync(() => WriteImageAsync(cleanedPath, image)).ConfigureAwait(false)) + .ToResult(exception => ScrapeErrorFactory.CreateImageSaveFailed(path, exception.Message)); + } + + private async Task WriteImageAsync(string cleanedPath, byte[] image) + { if (image.Length > 0) await fileSystem.File.WriteAllBytesAsync(cleanedPath, image).ConfigureAwait(false); + + return Unit.Value; } } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/SqliteConnectionStringProvider.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/SqliteConnectionStringProvider.cs index 85be6d1..5a6bfa6 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Support/SqliteConnectionStringProvider.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/SqliteConnectionStringProvider.cs @@ -1,3 +1,5 @@ +using System; +using System.IO; using AStar.Dev.Guard.Clauses; using Microsoft.Extensions.Configuration; @@ -11,13 +13,14 @@ namespace AStar.Dev.Wallpaper.Scrapper.Support; public static class SqliteConnectionStringProvider { /// - /// The connection string used when configuration does not supply ConnectionStrings:Sqlite. + /// The connection string used when configuration does not supply ScrapeConfiguration:ConnectionStrings:Sqlite. /// - public const string DefaultConnectionString = "Data Source=/home/jbarden/Documents/Scrapper/scrapper.db"; + public static readonly string DefaultConnectionString = + $"Data Source={Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "Documents", "Scrapper", "scrapper.db")}"; /// /// Resolves the Sqlite connection string from , falling back to /// when not configured. /// - public static string Get(IConfiguration configuration) => GuardAgainst.Null(configuration)["ConnectionStrings:Sqlite"] ?? DefaultConnectionString; + public static string Get(IConfiguration configuration) => GuardAgainst.Null(configuration)["ScrapeConfiguration:ConnectionStrings:Sqlite"] ?? DefaultConnectionString; } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Tags/TagsView.axaml.cs b/src/AStar.Dev.Wallpaper.Scrapper/Tags/TagsView.axaml.cs index d2caf05..51b38a1 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Tags/TagsView.axaml.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Tags/TagsView.axaml.cs @@ -1,4 +1,5 @@ using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Services; using AStar.Dev.Wallpaper.Scrapper.Support; using Avalonia.Controls; @@ -55,7 +56,7 @@ private async void OnImportTagsClicked(object? sender, RoutedEventArgs e) } ) .Tap(_ => logger.Information("Importing tags...")) - .Bind(_ => importExportService.ImportScrapedTagsFromFile()) + .Bind(_ => importExportService.ImportScrapedTagsFromFile().ToStringError()) .MapAsync(tags => scrapedTagService.ImportScrapedTagsAsync(tags, cts!.Token)) .TapAsync(_ => logger.Information("Tag import completed...")) .EnsureAsync(() => ResetUI()); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchWorkflow.cs b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchWorkflow.cs index e062582..94a5231 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchWorkflow.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchWorkflow.cs @@ -12,81 +12,98 @@ public sealed class SearchWorkflow(SearchResultsPage searchResultsPage, ScrapeCo { private SearchProgress progress = null!; - public async Task> RunAsync(ILogger scrapeLogger, CancellationToken ct = default) + public Task> RunAsync(CancellationToken ct = default) { - try - { - progress = SearchProgressFactory.Create(injectedScrapeConfiguration.SearchConfiguration, injectedScrapeConfiguration.ScrapeDirectories); - var searchCategories = SearchProgressFunctions.FilterSearchCategories(progress.SearchConfiguration, progress.SearchConfiguration.SearchCategories); - await ProcessSearchCategoriesAsync(searchCategories, scrapeLogger, ct); + progress = SearchProgressFactory.Create(injectedScrapeConfiguration.SearchConfiguration, injectedScrapeConfiguration.ScrapeDirectories); + var searchCategories = SearchProgressFunctions.FilterSearchCategories(progress.SearchConfiguration, progress.SearchConfiguration.SearchCategories); - return Unit.Value; - } - catch (Exception exception) when (exception is not OperationCanceledException) - { - scrapeLogger.Error(exception.GetBaseException().Message); - throw; - } + return ProcessSearchCategoriesAsync(searchCategories, ct).LogFailure(logger); } - private async Task ProcessSearchCategoriesAsync(IReadOnlyList searchCategories, ILogger scrapeLogger, CancellationToken ct) + private async Task> ProcessSearchCategoriesAsync(IReadOnlyList searchCategories, CancellationToken ct) { foreach (var searchCategory in searchCategories) { ct.ThrowIfCancellationRequested(); - string combinedSearchString = $"{progress.SearchConfiguration.SearchStringPrefix}{searchCategory.Id}{progress.SearchConfiguration.SearchStringSuffix}"; - progress = SearchProgressFunctions.UpdateSearchDetails(progress, combinedSearchString); + var categoryResult = await ProcessSearchCategoryAsync(searchCategory, ct).ConfigureAwait(false); + var categoryFailed = categoryResult.Match(_ => false, _ => true); - var pageDetails = await searchResultsPage.LoadSearchPageAsync(combinedSearchString, progress.SearchConfiguration.StartingPageNumber); + if (categoryFailed) return categoryResult; + } - if (pageDetails is { Ok: false, }) throw new InvalidOperationException("Could not get the image page after retry..."); + return Unit.Value; + } - var (pageCount, imageCount, subDirectoryName) = await searchResultsPage.PageInfoAsync(); - progress = SearchProgressFunctions.UpdateTotalPages(progress, pageCount); + private Task> ProcessSearchCategoryAsync(Category searchCategory, CancellationToken ct) + { + string combinedSearchString = $"{progress.SearchConfiguration.SearchStringPrefix}{searchCategory.Id}{progress.SearchConfiguration.SearchStringSuffix}"; + progress = SearchProgressFunctions.UpdateSearchDetails(progress, combinedSearchString); - if (searchCategory.IsUpToDate(imageCount, pageCount)) - { - logger.Information("{Category} is up to date (same image/page count), skipping...", searchCategory.Name); - await delayStrategy.DelayAsync(DelayKind.CategoryUpToDate, ct).ConfigureAwait(false); - continue; - } + return searchResultsPage.LoadSearchPageAsync(combinedSearchString, progress.SearchConfiguration.StartingPageNumber) + .BindAsync(_ => searchResultsPage.PageInfoAsync()) + .BindAsync(pageInfo => ProcessCategoryPageInfoAsync(searchCategory, combinedSearchString, pageInfo, ct)); + } - int startingPage = searchCategory.LastPageVisited > 0 ? searchCategory.LastPageVisited : 1; - progress = progress with { SearchConfiguration = progress.SearchConfiguration with { StartingPageNumber = startingPage, }, }; + private async Task> ProcessCategoryPageInfoAsync(Category searchCategory, string combinedSearchString, PageInfo pageInfo, CancellationToken ct) + { + progress = SearchProgressFunctions.UpdateTotalPages(progress, pageInfo.PageCount); - logger.Debug("Visiting {Category} from page {StartingPage} now...", searchCategory.Name, startingPage); - progress = SearchProgressFunctions.UpdateSubDirectory(progress, subDirectoryName); + if (searchCategory.IsUpToDate(pageInfo.ImageCount, pageInfo.PageCount)) + { + logger.Information("{Category} is up to date (same image/page count), skipping...", searchCategory.Name); + await delayStrategy.DelayAsync(DelayKind.CategoryUpToDate, ct).ConfigureAwait(false); - _ = directoryHelper.CreateDirectoryIfRequired([progress.ScrapeDirectories.RootDirectory.CombinePath(progress.ScrapeDirectories.BaseDirectory, subDirectoryName),]); + return Unit.Value; + } - await ProcessAllCategoryPagesAsync(searchCategory, combinedSearchString, scrapeLogger, ct); + int startingPage = searchCategory.LastPageVisited > 0 ? searchCategory.LastPageVisited : 1; + progress = progress with { SearchConfiguration = progress.SearchConfiguration with { StartingPageNumber = startingPage, }, }; - searchCategory.LastKnownImageCount = imageCount; - searchCategory.TotalPages = pageCount; - searchCategory.LastPageVisited = 0; - await configurationSaver.SaveUpdatedConfigurationAsync(); - } + logger.Debug("Visiting {Category} from page {StartingPage} now...", searchCategory.Name, startingPage); + progress = SearchProgressFunctions.UpdateSubDirectory(progress, pageInfo.SubDirectoryName); + + _ = directoryHelper.CreateDirectoryIfRequired([progress.ScrapeDirectories.RootDirectory.CombinePath(progress.ScrapeDirectories.BaseDirectory, pageInfo.SubDirectoryName),]); + + return await ProcessAllCategoryPagesAsync(searchCategory, combinedSearchString, ct) + .BindAsync(_ => SaveCategoryProgressAsync(searchCategory, pageInfo)) + .ConfigureAwait(false); } - private async Task ProcessAllCategoryPagesAsync(Category searchCategory, string combinedSearchString, ILogger scrapeLogger, CancellationToken ct) + private Task> SaveCategoryProgressAsync(Category searchCategory, PageInfo pageInfo) + { + searchCategory.LastKnownImageCount = pageInfo.ImageCount; + searchCategory.TotalPages = pageInfo.PageCount; + searchCategory.LastPageVisited = 0; + + return configurationSaver.SaveUpdatedConfigurationAsync(); + } + + private async Task> ProcessAllCategoryPagesAsync(Category searchCategory, string combinedSearchString, CancellationToken ct) { long startTimestamp = timeProvider.GetTimestamp(); - scrapeLogger.Debug("About to visit the specific {Category} pages now...", searchCategory.Name); + logger.Debug("About to visit the specific {Category} pages now...", searchCategory.Name); for (int currentPageNumber = progress.SearchConfiguration.StartingPageNumber; currentPageNumber <= progress.SearchConfiguration.TotalPages; currentPageNumber++) { await delayStrategy.DelayAsync(DelayKind.PageNavigation, ct).ConfigureAwait(false); - scrapeLogger.Debug("About to visit page {page} (of {totalPages}) for {Category} now...", currentPageNumber, progress.SearchConfiguration.TotalPages, searchCategory.Name); + logger.Debug("About to visit page {page} (of {totalPages}) for {Category} now...", currentPageNumber, progress.SearchConfiguration.TotalPages, searchCategory.Name); progress = progress with { SearchConfiguration = progress.SearchConfiguration with { StartingPageNumber = currentPageNumber, }, }; searchCategory.LastPageVisited = currentPageNumber; - await configurationSaver.SaveUpdatedConfigurationAsync(); - _ = await searchResultsPage.LoadSearchPageAsync(combinedSearchString, currentPageNumber); - var imagePageLinks = await searchResultsPage.ImagePageLinksAsync(); - await imagePageService.GetTheImagePagesAsync(imagePageLinks, searchCategory.Id, searchCategory.Name, ct); + var pageResult = await configurationSaver.SaveUpdatedConfigurationAsync() + .BindAsync(_ => searchResultsPage.LoadSearchPageAsync(combinedSearchString, currentPageNumber)) + .BindAsync(_ => searchResultsPage.ImagePageLinksAsync()) + .BindAsync(links => imagePageService.GetTheImagePagesAsync(links, searchCategory.Id, searchCategory.Name, ct)) + .ConfigureAwait(false); + + var pageFailed = pageResult.Match(_ => false, _ => true); + + if (pageFailed) return pageResult; } - scrapeLogger.Information("Completed visiting the {Category}. Total time: {CategoryVisitDuration}", searchCategory.Name, timeProvider.GetElapsedTime(startTimestamp)); + logger.Information("Completed visiting the {Category}. Total time: {CategoryVisitDuration}", searchCategory.Name, timeProvider.GetElapsedTime(startTimestamp)); + + return Unit.Value; } } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SubscriptionsWorkflow.cs b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SubscriptionsWorkflow.cs index 8985963..31c542d 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SubscriptionsWorkflow.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SubscriptionsWorkflow.cs @@ -1,8 +1,8 @@ +using AStar.Dev.FunctionalParadigm; using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Pages; using AStar.Dev.Wallpaper.Scrapper.Services; using AStar.Dev.Wallpaper.Scrapper.Support; -using Microsoft.Playwright; using Serilog.Core; namespace AStar.Dev.Wallpaper.Scrapper.Workflows; @@ -22,7 +22,7 @@ public async Task RunAsync(CancellationToken ct = default) { try { - await GetTheNewSubscriptionImagesAsync(ct); + await GetTheNewSubscriptionImagesAsync(ct).ConfigureAwait(false); } catch (Exception exception) when (exception is not OperationCanceledException) { @@ -34,17 +34,19 @@ public async Task RunAsync(CancellationToken ct = default) private async Task GetTheNewSubscriptionImagesAsync(CancellationToken ct) { _searchConfiguration = _searchConfiguration with { SubscriptionsStartingPageNumber = 1 }; - var pageDetails = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(_searchConfiguration.SubscriptionsStartingPageNumber); + var pageDetails = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(_searchConfiguration.SubscriptionsStartingPageNumber).ConfigureAwait(false); + var loadedSuccessfully = pageDetails.Match(_ => true, _ => false); - if (pageDetails is { Ok: false, }) _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(1); + if (!loadedSuccessfully) _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(1).ConfigureAwait(false); - var (pageCount, subDirectoryName) = await subscriptionsImagesListPage.PageInfoAsync(); + var pageInfo = (await subscriptionsImagesListPage.PageInfoAsync().ConfigureAwait(false)) + .Match(info => info, error => throw new InvalidOperationException(error.Message)); - if (subDirectoryName.Length > 0) _scrapeDirectories = _scrapeDirectories with { SubDirectoryName = subDirectoryName }; + if (pageInfo.SubDirectoryName.Length > 0) _scrapeDirectories = _scrapeDirectories with { SubDirectoryName = pageInfo.SubDirectoryName }; - UpdateSearchTotalPagesIfRequired(pageCount); + UpdateSearchTotalPagesIfRequired(pageInfo.PageCount); - await configurationSaver.SaveUpdatedConfigurationAsync(); + await configurationSaver.SaveUpdatedConfigurationAsync().ConfigureAwait(false); for (int currentPageNumber = _searchConfiguration.SubscriptionsStartingPageNumber; currentPageNumber <= _searchConfiguration.SubscriptionsTotalPages; @@ -52,20 +54,23 @@ private async Task GetTheNewSubscriptionImagesAsync(CancellationToken ct) { ct.ThrowIfCancellationRequested(); int delay = Random.Shared.Next(_searchConfiguration.ImagePauseInSeconds, _searchConfiguration.ImagePauseInSeconds + 4); - await Task.Delay(TimeSpan.FromSeconds(delay), ct); + await Task.Delay(TimeSpan.FromSeconds(delay), ct).ConfigureAwait(false); _searchConfiguration = _searchConfiguration with { SubscriptionsStartingPageNumber = currentPageNumber }; - await configurationSaver.SaveUpdatedConfigurationAsync(); + await configurationSaver.SaveUpdatedConfigurationAsync().ConfigureAwait(false); logger.Information("Getting page {subscriptionPage} (of {totalPagesForSubscriptions}) now.", currentPageNumber, _searchConfiguration.SubscriptionsTotalPages); - _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(currentPageNumber); - var imagePageLinks = await subscriptionsImagesListPage.GetImagePageLinksAsync(); + _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(currentPageNumber).ConfigureAwait(false); - await imagePageService.GetTheImagePagesAsync(imagePageLinks, "", subDirectoryName, ct: ct); + var imagePageLinks = (await subscriptionsImagesListPage.GetImagePageLinksAsync().ConfigureAwait(false)) + .Match(links => links, error => throw new InvalidOperationException(error.Message)); + + _ = (await imagePageService.GetTheImagePagesAsync(imagePageLinks, "", pageInfo.SubDirectoryName, ct: ct).ConfigureAwait(false)) + .Match(unit => unit, error => throw new InvalidOperationException(error.Message)); } - if (pageCount > 0) + if (pageInfo.PageCount > 0) { - _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(1); - await subscriptionsImagesListPage.ClearAsync(); + _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(1).ConfigureAwait(false); + _ = await subscriptionsImagesListPage.ClearAsync().ConfigureAwait(false); } } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/TopWallpapersWorkflow.cs b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/TopWallpapersWorkflow.cs index 1956b5f..5498e15 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/TopWallpapersWorkflow.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/TopWallpapersWorkflow.cs @@ -1,3 +1,4 @@ +using AStar.Dev.FunctionalParadigm; using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Pages; using AStar.Dev.Wallpaper.Scrapper.Services; @@ -19,7 +20,7 @@ public async Task RunAsync(CancellationToken ct = default) { try { - await GetTheNewTopWallpapersAsync(ct); + await GetTheNewTopWallpapersAsync(ct).ConfigureAwait(false); } catch (Exception exception) when (exception is not OperationCanceledException) { @@ -30,15 +31,17 @@ public async Task RunAsync(CancellationToken ct = default) private async Task GetTheNewTopWallpapersAsync(CancellationToken ct) { - var pageDetails = await topWallpapersPage.LoadTopWallpapersPageAsync(_searchConfiguration.TopWallpapersStartingPageNumber); + var pageDetails = await topWallpapersPage.LoadTopWallpapersPageAsync(_searchConfiguration.TopWallpapersStartingPageNumber).ConfigureAwait(false); + var loadedSuccessfully = pageDetails.Match(_ => true, _ => false); - if (pageDetails is { Ok: false, }) _ = await topWallpapersPage.LoadTopWallpapersPageAsync(1); + if (!loadedSuccessfully) _ = await topWallpapersPage.LoadTopWallpapersPageAsync(1).ConfigureAwait(false); - int pageCount = await topWallpapersPage.PageInfoAsync(); + int pageCount = (await topWallpapersPage.PageInfoAsync().ConfigureAwait(false)) + .Match(count => count, error => throw new InvalidOperationException(error.Message)); logger.Information("There are a total of {TopWallpapersPageCount} pages for the Top Wallpapers.", pageCount); _searchConfiguration = _searchConfiguration with { TopWallpapersTotalPages = pageCount }; - await configurationSaver.SaveUpdatedConfigurationAsync(); + await configurationSaver.SaveUpdatedConfigurationAsync().ConfigureAwait(false); for (int currentPageNumber = _searchConfiguration.TopWallpapersStartingPageNumber; currentPageNumber <= _searchConfiguration.TopWallpapersTotalPages; @@ -46,13 +49,16 @@ private async Task GetTheNewTopWallpapersAsync(CancellationToken ct) { ct.ThrowIfCancellationRequested(); int delay = Random.Shared.Next(_searchConfiguration.ImagePauseInSeconds, _searchConfiguration.ImagePauseInSeconds + 4); - await Task.Delay(TimeSpan.FromSeconds(delay), ct); + await Task.Delay(TimeSpan.FromSeconds(delay), ct).ConfigureAwait(false); _searchConfiguration = _searchConfiguration with { TopWallpapersStartingPageNumber = currentPageNumber }; - await configurationSaver.SaveUpdatedConfigurationAsync(); - _ = await topWallpapersPage.LoadTopWallpapersPageAsync(_searchConfiguration.TopWallpapersStartingPageNumber); - var imagePageLinks = await topWallpapersPage.GetImagePageLinksAsync(); + await configurationSaver.SaveUpdatedConfigurationAsync().ConfigureAwait(false); + _ = await topWallpapersPage.LoadTopWallpapersPageAsync(_searchConfiguration.TopWallpapersStartingPageNumber).ConfigureAwait(false); - await imagePageService.GetTheImagePagesAsync(imagePageLinks, "", "", ct: ct); + var imagePageLinks = (await topWallpapersPage.GetImagePageLinksAsync().ConfigureAwait(false)) + .Match(links => links, error => throw new InvalidOperationException(error.Message)); + + _ = (await imagePageService.GetTheImagePagesAsync(imagePageLinks, "", "", ct: ct).ConfigureAwait(false)) + .Match(unit => unit, error => throw new InvalidOperationException(error.Message)); } } } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/appsettings.json b/src/AStar.Dev.Wallpaper.Scrapper/appsettings.json index b1502d0..8ed5d86 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/appsettings.json +++ b/src/AStar.Dev.Wallpaper.Scrapper/appsettings.json @@ -1 +1,15 @@ -{"Logging":{"LogLevel":null,"Console":null,"Serilog":null}} +{ + "Logging": + { + "LogLevel":null, + "Console":null, + "Serilog":null + }, + "scrapeConfiguration": + { + "connectionStrings": + { + "sqlite": "Data Source=/some/source/folder/some-database-file.db" + } + } +} diff --git a/test/AStar.Dev.FunctionalParadigm.Tests.Unit/GivenRetryOnceAsync.cs b/test/AStar.Dev.FunctionalParadigm.Tests.Unit/GivenRetryOnceAsync.cs new file mode 100644 index 0000000..2e48554 --- /dev/null +++ b/test/AStar.Dev.FunctionalParadigm.Tests.Unit/GivenRetryOnceAsync.cs @@ -0,0 +1,159 @@ +using Shouldly; +using Xunit; + +namespace AStar.Dev.FunctionalParadigm.Tests.Unit; + +public sealed class GivenRetryOnceAsync +{ + [Fact] + public async Task when_the_first_attempt_succeeds_then_the_result_is_the_first_attempts_success() + { + var actual = await RetryExtensions.RetryOnceAsync( + () => Task.FromResult(Result.Success(1)), + () => Task.CompletedTask); + + actual.ShouldBe(new Ok(1)); + } + + [Fact] + public async Task when_the_first_attempt_succeeds_then_the_operation_is_invoked_exactly_once() + { + var attempts = 0; + + await RetryExtensions.RetryOnceAsync( + () => + { + attempts++; + + return Task.FromResult(Result.Success(1)); + }, + () => Task.CompletedTask); + + attempts.ShouldBe(1); + } + + [Fact] + public async Task when_the_first_attempt_succeeds_then_the_retry_callback_is_never_invoked() + { + var retryInvoked = false; + + await RetryExtensions.RetryOnceAsync( + () => Task.FromResult(Result.Success(1)), + () => + { + retryInvoked = true; + + return Task.CompletedTask; + }); + + retryInvoked.ShouldBeFalse(); + } + + [Fact] + public async Task when_the_first_attempt_fails_and_the_second_succeeds_then_the_result_is_the_second_attempts_success() + { + var attempts = 0; + + var actual = await RetryExtensions.RetryOnceAsync( + () => + { + attempts++; + + return Task.FromResult(attempts == 1 ? Result.Failure("first-failed") : Result.Success(2)); + }, + () => Task.CompletedTask); + + actual.ShouldBe(new Ok(2)); + } + + [Fact] + public async Task when_the_first_attempt_fails_and_the_second_succeeds_then_the_operation_is_invoked_exactly_twice() + { + var attempts = 0; + + await RetryExtensions.RetryOnceAsync( + () => + { + attempts++; + + return Task.FromResult(attempts == 1 ? Result.Failure("first-failed") : Result.Success(2)); + }, + () => Task.CompletedTask); + + attempts.ShouldBe(2); + } + + [Fact] + public async Task when_the_first_attempt_fails_and_the_second_succeeds_then_the_retry_callback_is_invoked_exactly_once() + { + var retryInvocations = 0; + var attempts = 0; + + await RetryExtensions.RetryOnceAsync( + () => + { + attempts++; + + return Task.FromResult(attempts == 1 ? Result.Failure("first-failed") : Result.Success(2)); + }, + () => + { + retryInvocations++; + + return Task.CompletedTask; + }); + + retryInvocations.ShouldBe(1); + } + + [Fact] + public async Task when_both_attempts_fail_then_the_result_is_the_second_attempts_failure() + { + var attempts = 0; + + var actual = await RetryExtensions.RetryOnceAsync( + () => + { + attempts++; + + return Task.FromResult(Result.Failure($"failed-{attempts}")); + }, + () => Task.CompletedTask); + + actual.ShouldBe(new Fail("failed-2")); + } + + [Fact] + public async Task when_both_attempts_fail_then_the_operation_is_invoked_exactly_twice() + { + var attempts = 0; + + await RetryExtensions.RetryOnceAsync( + () => + { + attempts++; + + return Task.FromResult(Result.Failure($"failed-{attempts}")); + }, + () => Task.CompletedTask); + + attempts.ShouldBe(2); + } + + [Fact] + public async Task when_both_attempts_fail_then_the_retry_callback_is_invoked_exactly_once() + { + var retryInvocations = 0; + + await RetryExtensions.RetryOnceAsync( + () => Task.FromResult(Result.Failure("failed")), + () => + { + retryInvocations++; + + return Task.CompletedTask; + }); + + retryInvocations.ShouldBe(1); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/DTOs/GivenTheFileClassificationDtoValidator.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/DTOs/GivenTheFileClassificationDtoValidator.cs new file mode 100644 index 0000000..17dfe1a --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/DTOs/GivenTheFileClassificationDtoValidator.cs @@ -0,0 +1,70 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.DTOs; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.DTOs; + +public sealed class GivenTheFileClassificationDtoValidator +{ + private static FileClassification BuildDto(string name, int level) => new() { Name = name, Level = level, }; + + [Fact] + public void when_the_name_and_level_are_valid_then_the_result_is_valid() => + FileClassificationDtoValidator.Validate(BuildDto("Animals", 3), 0).ShouldBeOfType>(); + + [Fact] + public void when_the_name_is_empty_then_the_result_is_invalid() => + FileClassificationDtoValidator.Validate(BuildDto("", 3), 0).ShouldBeOfType>(); + + [Fact] + public void when_the_name_is_empty_then_the_error_property_identifies_the_row_and_field() => + FileClassificationDtoValidator.Validate(BuildDto("", 3), 2).ShouldBeOfType>() + .Errors.ShouldHaveSingleItem().Property.ShouldBe("Categories[2].Name"); + + [Fact] + public void when_the_level_is_below_the_minimum_then_the_result_is_invalid() => + FileClassificationDtoValidator.Validate(BuildDto("Animals", 0), 0).ShouldBeOfType>(); + + [Fact] + public void when_the_level_is_above_the_maximum_then_the_result_is_invalid() => + FileClassificationDtoValidator.Validate(BuildDto("Animals", 4), 0).ShouldBeOfType>(); + + [Fact] + public void when_the_level_is_invalid_then_the_error_property_identifies_the_row_and_field() => + FileClassificationDtoValidator.Validate(BuildDto("Animals", 9), 1).ShouldBeOfType>() + .Errors.ShouldHaveSingleItem().Property.ShouldBe("Categories[1].Level"); + + [Fact] + public void when_both_the_name_and_level_are_invalid_then_both_errors_are_accumulated() => + FileClassificationDtoValidator.Validate(BuildDto("", 0), 0).ShouldBeOfType>() + .Errors.Count.ShouldBe(2); + + [Fact] + public void when_validating_all_rows_and_all_are_valid_then_the_result_is_valid() + { + FileClassification[] dtos = [BuildDto("Animals", 3), BuildDto("Cars", 2),]; + + FileClassificationDtoValidator.ValidateAll(dtos).ShouldBeOfType>>(); + } + + [Fact] + public void when_validating_all_rows_and_three_rows_are_invalid_then_all_three_errors_are_reported() + { + FileClassification[] dtos = [BuildDto("", 3), BuildDto("Valid", 0), BuildDto("Another Valid", 9),]; + + var errors = FileClassificationDtoValidator.ValidateAll(dtos).ShouldBeOfType>>().Errors; + + errors.Count.ShouldBe(3); + } + + [Fact] + public void when_validating_all_rows_and_three_rows_are_invalid_then_the_errors_are_reported_in_row_order() + { + FileClassification[] dtos = [BuildDto("", 3), BuildDto("Valid", 0), BuildDto("Another Valid", 9),]; + + var errors = FileClassificationDtoValidator.ValidateAll(dtos).ShouldBeOfType>>().Errors; + + errors[0].Property.ShouldBe("Categories[0].Name"); + errors[1].Property.ShouldBe("Categories[1].Level"); + errors[2].Property.ShouldBe("Categories[2].Level"); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenTheImagePageOutcomeFactory.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenTheImagePageOutcomeFactory.cs new file mode 100644 index 0000000..81a09ae --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenTheImagePageOutcomeFactory.cs @@ -0,0 +1,37 @@ +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Repositories; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Models; + +public sealed class GivenTheImagePageOutcomeFactory +{ + private static readonly TagData[] RawTags = [new("Sports Car", "Vehicles > Cars & Motorcycles"),]; + + [Fact] + public void when_creating_a_scraped_image_then_the_image_url_is_preserved() => + ImagePageOutcomeFactory.CreateScrapedImage("https://example.test/image.jpg", ["dir",], "prefix", ["tag",], RawTags).ImageUrl.ShouldBe("https://example.test/image.jpg"); + + [Fact] + public void when_creating_a_scraped_image_then_the_directory_segments_are_preserved() => + ImagePageOutcomeFactory.CreateScrapedImage("https://example.test/image.jpg", ["dir",], "prefix", ["tag",], RawTags).DirectorySegments.ShouldBe(["dir",]); + + [Fact] + public void when_creating_a_scraped_image_then_the_file_prefix_is_preserved() => + ImagePageOutcomeFactory.CreateScrapedImage("https://example.test/image.jpg", ["dir",], "prefix", ["tag",], RawTags).FilePrefix.ShouldBe("prefix"); + + [Fact] + public void when_creating_a_scraped_image_then_the_tags_are_preserved() => + ImagePageOutcomeFactory.CreateScrapedImage("https://example.test/image.jpg", ["dir",], "prefix", ["tag",], RawTags).Tags.ShouldBe(["tag",]); + + [Fact] + public void when_creating_a_scraped_image_then_the_raw_tags_are_preserved() => + ImagePageOutcomeFactory.CreateScrapedImage("https://example.test/image.jpg", ["dir",], "prefix", ["tag",], RawTags).RawTags.ShouldBe(RawTags); + + [Fact] + public void when_creating_a_skipped_image_then_the_tags_are_preserved() => + ImagePageOutcomeFactory.CreateSkippedImage(["ignored-tag",], RawTags).Tags.ShouldBe(["ignored-tag",]); + + [Fact] + public void when_creating_a_skipped_image_then_the_raw_tags_are_preserved() => + ImagePageOutcomeFactory.CreateSkippedImage(["ignored-tag",], RawTags).RawTags.ShouldBe(RawTags); +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenTheScrapeErrorFactory.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenTheScrapeErrorFactory.cs index dba15ca..f725ef0 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenTheScrapeErrorFactory.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Models/GivenTheScrapeErrorFactory.cs @@ -1,3 +1,4 @@ +using AStar.Dev.FunctionalParadigm; using AStar.Dev.Wallpaper.Scrapper.Models; namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Models; @@ -47,4 +48,36 @@ public void when_creating_an_unexpected_error_then_the_exception_is_preserved() [Fact] public void when_creating_an_unexpected_error_then_the_message_is_the_exception_message() => ScrapeErrorFactory.CreateUnexpectedError(new InvalidOperationException("boom")).Message.ShouldBe("boom"); + + [Fact] + public void when_creating_an_image_dimension_read_failed_error_then_the_path_is_preserved() => + ScrapeErrorFactory.CreateImageDimensionReadFailed("/some/path/image.jpg", "could not read dimensions").Path.ShouldBe("/some/path/image.jpg"); + + [Fact] + public void when_creating_an_image_dimension_read_failed_error_then_the_message_is_preserved() => + ScrapeErrorFactory.CreateImageDimensionReadFailed("/some/path/image.jpg", "could not read dimensions").Message.ShouldBe("could not read dimensions"); + + [Fact] + public void when_creating_an_import_failed_error_then_the_file_path_is_preserved() => + ScrapeErrorFactory.CreateImportFailed("/some/path/export.json", "file not found").FilePath.ShouldBe("/some/path/export.json"); + + [Fact] + public void when_creating_an_import_failed_error_then_the_message_is_preserved() => + ScrapeErrorFactory.CreateImportFailed("/some/path/export.json", "file not found").Message.ShouldBe("file not found"); + + [Fact] + public void when_creating_a_validation_failed_error_then_the_errors_are_preserved() + { + ValidationError[] errors = [new("Name", "Name is required"), new("Level", "Level must be between 1 and 3"),]; + + ScrapeErrorFactory.CreateValidationFailed(errors, "validation failed").Errors.ShouldBe(errors); + } + + [Fact] + public void when_creating_a_validation_failed_error_then_the_message_is_preserved() + { + ValidationError[] errors = [new("Name", "Name is required"),]; + + ScrapeErrorFactory.CreateValidationFailed(errors, "validation failed").Message.ShouldBe("validation failed"); + } } diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenASearchResultsPage.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenASearchResultsPage.cs new file mode 100644 index 0000000..32b31b6 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenASearchResultsPage.cs @@ -0,0 +1,173 @@ +using NSubstitute.ExceptionExtensions; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Pages; +using AStar.Dev.Wallpaper.Scrapper.Services; +using Microsoft.Playwright; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Pages; + +public sealed class GivenASearchResultsPage +{ + private const string SearchString = "https://example.test/search/cars/?page="; + private const int PageNumber = 3; + + private static SearchResultsPage BuildSut(IPage page) + { + var playwrightService = Substitute.For(); + playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + + return new SearchResultsPage(playwrightService); + } + + private static IPage BuildPageReturning(IResponse? response) + { + var page = Substitute.For(); + page.GotoAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(response)); + + return page; + } + + [Fact] + public async Task when_the_first_navigation_attempt_succeeds_then_the_result_is_ok() + { + var okResponse = Substitute.For(); + okResponse.Ok.Returns(true); + var page = BuildPageReturning(okResponse); + var sut = BuildSut(page); + + var result = await sut.LoadSearchPageAsync(SearchString, PageNumber); + + result.ShouldBeOfType>(); + } + + [Fact] + public async Task when_the_first_navigation_attempt_succeeds_then_goto_async_is_called_exactly_once() + { + var okResponse = Substitute.For(); + okResponse.Ok.Returns(true); + var page = BuildPageReturning(okResponse); + var sut = BuildSut(page); + + await sut.LoadSearchPageAsync(SearchString, PageNumber); + + await page.Received(1).GotoAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task when_both_navigation_attempts_report_not_ok_then_goto_async_is_called_exactly_twice() + { + var notOkResponse = Substitute.For(); + notOkResponse.Ok.Returns(false); + var page = BuildPageReturning(notOkResponse); + var sut = BuildSut(page); + + await sut.LoadSearchPageAsync(SearchString, PageNumber); + + await page.Received(2).GotoAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task when_both_navigation_attempts_report_not_ok_then_a_page_load_failed_error_is_returned() + { + var notOkResponse = Substitute.For(); + notOkResponse.Ok.Returns(false); + var page = BuildPageReturning(notOkResponse); + var sut = BuildSut(page); + + var result = await sut.LoadSearchPageAsync(SearchString, PageNumber); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_both_navigation_attempts_report_not_ok_then_the_page_load_failed_error_carries_the_full_url() + { + var notOkResponse = Substitute.For(); + notOkResponse.Ok.Returns(false); + var page = BuildPageReturning(notOkResponse); + var sut = BuildSut(page); + + var result = await sut.LoadSearchPageAsync(SearchString, PageNumber); + + result.ShouldBeOfType>().Error.ShouldBeOfType().Url.ShouldBe($"{SearchString}{PageNumber}"); + } + + [Fact] + public async Task when_navigation_throws_then_a_page_load_failed_error_is_returned() + { + var page = Substitute.For(); + page.GotoAsync(Arg.Any(), Arg.Any()).ThrowsAsync(new PlaywrightException("navigation failed")); + var sut = BuildSut(page); + + var result = await sut.LoadSearchPageAsync(SearchString, PageNumber); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_the_header_reports_a_wallpaper_count_then_the_page_info_is_parsed_successfully() + { + var headerLocator = Substitute.For(); + headerLocator.TextContentAsync().Returns(Task.FromResult("48 Wallpapers found for #Cars")); + var page = Substitute.For(); + page.GetByText(Arg.Is(text => text.Contains("Wallpapers found")), Arg.Any()).Returns(headerLocator); + var sut = BuildSut(page); + + var result = await sut.PageInfoAsync(); + + result.ShouldBeOfType>().Value.SubDirectoryName.ShouldBe("Cars"); + } + + [Fact] + public async Task when_the_header_text_is_missing_then_a_page_parse_failed_error_is_returned() + { + var headerLocator = Substitute.For(); + headerLocator.TextContentAsync().Returns(Task.FromResult(null)); + var page = Substitute.For(); + page.GetByText(Arg.Any(), Arg.Any()).Returns(headerLocator); + var sut = BuildSut(page); + + var result = await sut.PageInfoAsync(); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_reading_the_header_throws_then_a_page_load_failed_error_is_returned() + { + var page = Substitute.For(); + page.GetByText(Arg.Any(), Arg.Any()).Returns(_ => throw new PlaywrightException("locator failed")); + var sut = BuildSut(page); + + var result = await sut.PageInfoAsync(); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_there_are_no_image_previews_then_the_links_result_is_ok_and_empty() + { + var linksLocator = Substitute.For(); + linksLocator.AllAsync().Returns(Task.FromResult>([])); + var page = Substitute.For(); + page.GetByRole(AriaRole.Link, Arg.Any()).Returns(linksLocator); + var sut = BuildSut(page); + + var result = await sut.ImagePageLinksAsync(); + + result.ShouldBeOfType, ScrapeError>>().Value.ShouldBeEmpty(); + } + + [Fact] + public async Task when_reading_the_image_previews_throws_then_a_page_load_failed_error_is_returned() + { + var page = Substitute.For(); + page.GetByRole(AriaRole.Link, Arg.Any()).Returns(_ => throw new PlaywrightException("locator failed")); + var sut = BuildSut(page); + + var result = await sut.ImagePageLinksAsync(); + + result.ShouldBeOfType, ScrapeError>>().Error.ShouldBeOfType(); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenASubscriptionsImagesListPage.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenASubscriptionsImagesListPage.cs new file mode 100644 index 0000000..143e747 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenASubscriptionsImagesListPage.cs @@ -0,0 +1,142 @@ +using NSubstitute.ExceptionExtensions; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Pages; +using AStar.Dev.Wallpaper.Scrapper.Services; +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; +using Microsoft.Playwright; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Pages; + +public sealed class GivenASubscriptionsImagesListPage +{ + private static SubscriptionsImagesListPage BuildSut(IPage page) + { + var playwrightService = Substitute.For(); + playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + var searchConfiguration = new SearchConfigurationBuilder().Build(); + + return new SubscriptionsImagesListPage(playwrightService, searchConfiguration); + } + + [Fact] + public async Task when_loading_the_subscriptions_page_succeeds_then_the_result_is_ok() + { + var response = Substitute.For(); + var page = Substitute.For(); + page.GotoAsync(Arg.Any()).Returns(Task.FromResult(response)); + var sut = BuildSut(page); + + var result = await sut.LoadSubscriptionResultsPageAsync(1); + + result.ShouldBeOfType>(); + } + + [Fact] + public async Task when_loading_the_subscriptions_page_and_navigation_throws_then_a_page_load_failed_error_is_returned() + { + var page = Substitute.For(); + page.GotoAsync(Arg.Any()).ThrowsAsync(new PlaywrightException("navigation failed")); + var sut = BuildSut(page); + + var result = await sut.LoadSubscriptionResultsPageAsync(1); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_the_header_reports_new_subscription_wallpapers_then_the_page_info_is_parsed_successfully() + { + var headerLocator = Substitute.For(); + headerLocator.TextContentAsync().Returns(Task.FromResult("24 New Subscription Wallpapers")); + var page = Substitute.For(); + page.GetByText(Arg.Any(), Arg.Any()).Returns(headerLocator); + var sut = BuildSut(page); + + var result = await sut.PageInfoAsync(); + + result.ShouldBeOfType>(); + } + + [Fact] + public async Task when_the_subscriptions_header_text_is_missing_then_a_page_parse_failed_error_is_returned() + { + var headerLocator = Substitute.For(); + headerLocator.TextContentAsync().Returns(Task.FromResult(null)); + var page = Substitute.For(); + page.GetByText(Arg.Any(), Arg.Any()).Returns(headerLocator); + var sut = BuildSut(page); + + var result = await sut.PageInfoAsync(); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_reading_the_subscriptions_header_throws_then_a_page_load_failed_error_is_returned() + { + var page = Substitute.For(); + page.GetByText(Arg.Any(), Arg.Any()).Returns(_ => throw new PlaywrightException("locator failed")); + var sut = BuildSut(page); + + var result = await sut.PageInfoAsync(); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_there_are_no_image_previews_then_the_links_result_is_ok_and_empty() + { + var linksLocator = Substitute.For(); + linksLocator.AllAsync().Returns(Task.FromResult>([])); + var page = Substitute.For(); + page.GetByRole(AriaRole.Link).Returns(linksLocator); + var sut = BuildSut(page); + + var result = await sut.GetImagePageLinksAsync(); + + result.ShouldBeOfType, ScrapeError>>().Value.ShouldBeEmpty(); + } + + [Fact] + public async Task when_reading_the_image_previews_throws_then_a_page_load_failed_error_is_returned() + { + var page = Substitute.For(); + page.GetByRole(AriaRole.Link).Returns(_ => throw new PlaywrightException("locator failed")); + var sut = BuildSut(page); + + var result = await sut.GetImagePageLinksAsync(); + + result.ShouldBeOfType, ScrapeError>>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_clearing_subscriptions_succeeds_then_the_result_is_ok() + { + var clearLink = Substitute.For(); + clearLink.ClickAsync().Returns(Task.CompletedTask); + var filteredLocator = Substitute.For(); + filteredLocator.GetByRole(AriaRole.Link, Arg.Any()).Returns(clearLink); + var divLocator = Substitute.For(); + divLocator.Filter(Arg.Any()).Returns(filteredLocator); + var page = Substitute.For(); + page.Locator("div").Returns(divLocator); + var sut = BuildSut(page); + + var result = await sut.ClearAsync(); + + result.ShouldBeOfType>(); + } + + [Fact] + public async Task when_clearing_subscriptions_throws_then_a_page_load_failed_error_is_returned() + { + var page = Substitute.For(); + page.Locator("div").Returns(_ => throw new PlaywrightException("locator failed")); + var sut = BuildSut(page); + + var result = await sut.ClearAsync(); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenATopWallpapersPage.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenATopWallpapersPage.cs index cc49fa2..e600171 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenATopWallpapersPage.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenATopWallpapersPage.cs @@ -1,3 +1,6 @@ +using NSubstitute.ExceptionExtensions; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Pages; using AStar.Dev.Wallpaper.Scrapper.Services; using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; @@ -7,43 +10,105 @@ namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Pages; public sealed class GivenATopWallpapersPage { + private static TopWallpapersPage BuildSut(IPage page) + { + var playwrightService = Substitute.For(); + playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + var searchConfiguration = new SearchConfigurationBuilder().Build(); + + return new TopWallpapersPage(playwrightService, searchConfiguration); + } + [Fact] - public async Task when_loading_the_top_wallpapers_page_for_the_first_time_then_no_exception_is_thrown() + public async Task when_loading_the_top_wallpapers_page_for_the_first_time_then_the_result_is_ok() { var response = Substitute.For(); - var page = Substitute.For(); page.GotoAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(response)); + var sut = BuildSut(page); - var playwrightService = Substitute.For(); - playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + var result = await sut.LoadTopWallpapersPageAsync(1); - var searchConfiguration = new SearchConfigurationBuilder().Build(); - var sut = new TopWallpapersPage(playwrightService, searchConfiguration); + result.ShouldBeOfType>(); + } - var exception = await Record.ExceptionAsync(() => sut.LoadTopWallpapersPageAsync(1)); + [Fact] + public async Task when_loading_the_top_wallpapers_page_and_navigation_throws_then_a_page_load_failed_error_is_returned() + { + var page = Substitute.For(); + page.GotoAsync(Arg.Any(), Arg.Any()).ThrowsAsync(new PlaywrightException("navigation failed")); + var sut = BuildSut(page); - exception.ShouldBeNull(); + var result = await sut.LoadTopWallpapersPageAsync(1); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); } [Fact] - public async Task when_getting_the_page_info_for_the_first_time_then_no_exception_is_thrown() + public async Task when_getting_the_page_info_for_the_first_time_then_the_page_count_is_parsed() { var locator = Substitute.For(); locator.First.Returns(locator); locator.TextContentAsync().Returns(Task.FromResult("Page 1 / 5")); + var page = Substitute.For(); + page.GetByText(Arg.Any(), Arg.Any()).Returns(locator); + var sut = BuildSut(page); + var result = await sut.PageInfoAsync(); + + result.ShouldBeOfType>().Value.ShouldBe(5); + } + + [Fact] + public async Task when_the_page_info_header_text_is_missing_then_a_page_parse_failed_error_is_returned() + { + var locator = Substitute.For(); + locator.First.Returns(locator); + locator.TextContentAsync().Returns(Task.FromResult(null)); var page = Substitute.For(); page.GetByText(Arg.Any(), Arg.Any()).Returns(locator); + var sut = BuildSut(page); - var playwrightService = Substitute.For(); - playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + var result = await sut.PageInfoAsync(); - var searchConfiguration = new SearchConfigurationBuilder().Build(); - var sut = new TopWallpapersPage(playwrightService, searchConfiguration); + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_reading_the_page_info_header_throws_then_a_page_load_failed_error_is_returned() + { + var page = Substitute.For(); + page.GetByText(Arg.Any(), Arg.Any()).Returns(_ => throw new PlaywrightException("locator failed")); + var sut = BuildSut(page); + + var result = await sut.PageInfoAsync(); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_there_are_no_image_previews_then_the_links_result_is_ok_and_empty() + { + var linksLocator = Substitute.For(); + linksLocator.AllAsync().Returns(Task.FromResult>([])); + var page = Substitute.For(); + page.GetByRole(AriaRole.Link, Arg.Any()).Returns(linksLocator); + var sut = BuildSut(page); + + var result = await sut.GetImagePageLinksAsync(); + + result.ShouldBeOfType, ScrapeError>>().Value.ShouldBeEmpty(); + } + + [Fact] + public async Task when_reading_the_image_previews_throws_then_a_page_load_failed_error_is_returned() + { + var page = Substitute.For(); + page.GetByRole(AriaRole.Link, Arg.Any()).Returns(_ => throw new PlaywrightException("locator failed")); + var sut = BuildSut(page); - var exception = await Record.ExceptionAsync(() => sut.PageInfoAsync()); + var result = await sut.GetImagePageLinksAsync(); - exception.ShouldBeNull(); + result.ShouldBeOfType, ScrapeError>>().Error.ShouldBeOfType(); } } diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenAnImagePage.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenAnImagePage.cs index c07a8df..66d2d8b 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenAnImagePage.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Pages/GivenAnImagePage.cs @@ -1,3 +1,7 @@ +using NSubstitute.ExceptionExtensions; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.DTOs; +using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Pages; using AStar.Dev.Wallpaper.Scrapper.Repositories; using AStar.Dev.Wallpaper.Scrapper.Services; @@ -10,34 +14,111 @@ public sealed class GivenAnImagePage { private const string Link = "https://example.test/w/12345"; private const string CategoryName = "Cars"; + private const string ImageUrl = "https://example.test/images/12345.jpg"; - [Fact] - public async Task when_getting_the_image_from_the_page_then_the_scraped_tag_repository_is_saved_to_exactly_once() + private static ImagePage BuildSut(IPage page, TagsToIgnoreCompletely? tagsToIgnoreCompletely = null, TagsTextToIgnore? tagsTextToIgnore = null) + { + var playwrightService = Substitute.For(); + playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + var scrapeConfiguration = new ScrapeConfigurationBuilder().Build(); + + return new ImagePage(playwrightService, scrapeConfiguration, tagsToIgnoreCompletely ?? new(), tagsTextToIgnore ?? new()); + } + + private static IPage BuildPage(string tagText, string? tagCategory, string? imageSrc) { var tagLocator = Substitute.For(); - tagLocator.InnerTextAsync().Returns(Task.FromResult("Sports Car")); - tagLocator.GetAttributeAsync("original-title").Returns(Task.FromResult("Vehicles > Cars & Motorcycles")); + tagLocator.InnerTextAsync().Returns(Task.FromResult(tagText)); + tagLocator.GetAttributeAsync("original-title").Returns(Task.FromResult(tagCategory)); var tagsLocator = Substitute.For(); tagsLocator.AllAsync().Returns(Task.FromResult>([tagLocator,])); var imageLocator = Substitute.For(); - imageLocator.GetAttributeAsync("src").Returns(Task.FromResult("https://example.test/images/12345.jpg")); + imageLocator.GetAttributeAsync("src").Returns(Task.FromResult(imageSrc)); var page = Substitute.For(); page.Locator(".tagname", Arg.Any()).Returns(tagsLocator); page.Locator("#wallpaper", Arg.Any()).Returns(imageLocator); - var playwrightService = Substitute.For(); - playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + return page; + } - var scrapedTagRepository = Substitute.For(); - var scrapeConfiguration = new ScrapeConfigurationBuilder().Build(); + [Fact] + public async Task when_the_tags_are_accepted_then_the_outcome_is_a_scraped_image() + { + var sut = BuildSut(BuildPage("Sports Car", "Vehicles > Cars & Motorcycles", ImageUrl)); + + var result = await sut.GetImageFromPageAsync(Link, CategoryName); + + result.ShouldBeOfType>().Value.ShouldBeOfType(); + } + + [Fact] + public async Task when_the_tags_are_accepted_then_the_scraped_image_carries_the_image_url() + { + var sut = BuildSut(BuildPage("Sports Car", "Vehicles > Cars & Motorcycles", ImageUrl)); + + var result = await sut.GetImageFromPageAsync(Link, CategoryName); + + result.ShouldBeOfType>().Value.ShouldBeOfType().ImageUrl.ShouldBe(ImageUrl); + } + + [Fact] + public async Task when_the_tags_are_accepted_then_the_scraped_image_carries_the_raw_tag_data() + { + var sut = BuildSut(BuildPage("Sports Car", "Vehicles > Cars & Motorcycles", ImageUrl)); - var sut = new ImagePage(playwrightService, scrapeConfiguration, new(), new(), scrapedTagRepository); + var result = await sut.GetImageFromPageAsync(Link, CategoryName); + + var scraped = result.ShouldBeOfType>().Value.ShouldBeOfType(); + scraped.RawTags.ShouldBe([new TagData("Sports Car", "Vehicles > Cars & Motorcycles"),]); + } + + [Fact] + public async Task when_a_tag_category_matches_the_ignore_completely_list_then_the_outcome_is_a_skipped_image() + { + var tagsToIgnoreCompletely = new TagsToIgnoreCompletely { Tags = ["ExcludedCategory",], }; + var sut = BuildSut(BuildPage("BadCategory", "ExcludedCategory", ImageUrl), tagsToIgnoreCompletely); + + var result = await sut.GetImageFromPageAsync(Link, CategoryName); + + result.ShouldBeOfType>().Value.ShouldBeOfType(); + } + + [Fact] + public async Task when_a_tag_category_matches_the_ignore_completely_list_then_the_skipped_image_carries_the_raw_tag_data() + { + var tagsToIgnoreCompletely = new TagsToIgnoreCompletely { Tags = ["ExcludedCategory",], }; + var sut = BuildSut(BuildPage("BadCategory", "ExcludedCategory", ImageUrl), tagsToIgnoreCompletely); + + var result = await sut.GetImageFromPageAsync(Link, CategoryName); + + var skipped = result.ShouldBeOfType>().Value.ShouldBeOfType(); + skipped.RawTags.ShouldBe([new TagData("BadCategory", "ExcludedCategory"),]); + } + + [Fact] + public async Task when_navigating_to_the_page_throws_then_a_page_load_failed_error_is_returned() + { + var page = Substitute.For(); + page.GotoAsync(Link, Arg.Any()).ThrowsAsync(new PlaywrightException("navigation failed")); + var sut = BuildSut(page); + + var result = await sut.GetImageFromPageAsync(Link, CategoryName); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_navigating_to_the_page_throws_then_the_page_load_failed_error_carries_the_link_as_the_url() + { + var page = Substitute.For(); + page.GotoAsync(Link, Arg.Any()).ThrowsAsync(new PlaywrightException("navigation failed")); + var sut = BuildSut(page); - await sut.GetImageFromPageAsync(Link, CategoryName); + var result = await sut.GetImageFromPageAsync(Link, CategoryName); - await scrapedTagRepository.Received(1).SaveAsync(Arg.Any>()); + result.ShouldBeOfType>().Error.ShouldBeOfType().Url.ShouldBe(Link); } } diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAFileClassificationService.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAFileClassificationService.cs index 5899ce4..7728984 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAFileClassificationService.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAFileClassificationService.cs @@ -1,5 +1,7 @@ +using AStar.Dev.FunctionalParadigm; using AStar.Dev.Infrastructure.AppDb; using AStar.Dev.Infrastructure.AppDb.Entities; +using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Services; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; @@ -498,6 +500,50 @@ public async Task when_importing_a_classification_whose_keyword_already_exists_w count.ShouldBe(1); } + [Fact] + public async Task when_classifying_with_a_matching_image_tag_then_the_result_is_ok() + { + var fileDetail = new FileDetailEntity + { + FileName = new FileName("test.jpg"), + DirectoryName = new DirectoryName("/tmp"), + FileHandle = FileHandleFactory.Create("test-handle-result-ok") + }; + await using var seedCtx = new AppDbContext(options); + seedCtx.Files.Add(fileDetail); + seedCtx.ScrapedTags.Add(new ScrapedTagEntity { Value = "outdoors", IncludeInSearch = true }); + await seedCtx.SaveChangesAsync(TestContext.Current.CancellationToken); + + var pageData = await sut.LoadPageClassificationDataAsync("any-category", TestContext.Current.CancellationToken); + + var result = await sut.ClassifyAsync(fileDetail, pageData, ["outdoors"], TestContext.Current.CancellationToken); + + result.ShouldBeOfType>(); + } + + [Fact] + public async Task when_classifying_with_a_tag_matching_no_existing_category_then_the_junction_row_links_to_the_newly_created_category() + { + var fileDetail = new FileDetailEntity + { + FileName = new FileName("test.jpg"), + DirectoryName = new DirectoryName("/tmp"), + FileHandle = FileHandleFactory.Create("test-handle-junction-links-new-category") + }; + await using var seedCtx = new AppDbContext(options); + seedCtx.Files.Add(fileDetail); + seedCtx.ScrapedTags.Add(new ScrapedTagEntity { Value = "brand new junction tag", IncludeInSearch = true }); + await seedCtx.SaveChangesAsync(TestContext.Current.CancellationToken); + + var pageData = await sut.LoadPageClassificationDataAsync("any-category", TestContext.Current.CancellationToken); + await sut.ClassifyAsync(fileDetail, pageData, ["brand new junction tag"], TestContext.Current.CancellationToken); + + await using var verifyCtx = new AppDbContext(options); + var created = await verifyCtx.FileClassificationCategories.SingleAsync(c => c.Name == "Brand New Junction Tag", TestContext.Current.CancellationToken); + var junction = await verifyCtx.FileClassifications.SingleAsync(TestContext.Current.CancellationToken); + junction.CategoryId.ShouldBe(created.Id); + } + private static ScrapeConfigurationEntity CreateScrapeConfigEntity() => new() { ConnectionStrings = new ConnectionStringsEntity { Sqlite = "Data Source=test.db" }, diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImagePageServiceWithADelayStrategy.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImagePageServiceWithADelayStrategy.cs index 35b37d0..1f52696 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImagePageServiceWithADelayStrategy.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImagePageServiceWithADelayStrategy.cs @@ -36,11 +36,11 @@ public async ValueTask InitializeAsync() public async ValueTask DisposeAsync() => await connection.DisposeAsync(); - private static ImagePage BuildImagePageReturningImage(string imageUrl) + private static ImagePage BuildImagePageReturningScrapedImage(string imageUrl, string tagText = "Nature", string? tagCategory = "Landscape") { var tagLocator = Substitute.For(); - tagLocator.InnerTextAsync().Returns(Task.FromResult("Nature")); - tagLocator.GetAttributeAsync("original-title").Returns(Task.FromResult("Landscape")); + tagLocator.InnerTextAsync().Returns(Task.FromResult(tagText)); + tagLocator.GetAttributeAsync("original-title").Returns(Task.FromResult(tagCategory)); var tagsLocator = Substitute.For(); tagsLocator.AllAsync().Returns(Task.FromResult>([tagLocator,])); @@ -57,17 +57,77 @@ private static ImagePage BuildImagePageReturningImage(string imageUrl) var scrapeConfiguration = new ScrapeConfigurationBuilder().Build(); - return new ImagePage(playwrightService, scrapeConfiguration, new(), new(), Substitute.For()); + return new ImagePage(playwrightService, scrapeConfiguration, new(), new()); } - private ImagePageService BuildService(ImagePage imagePage, IFileDetailRepository fileDetailRepository, IDelayStrategy delayStrategy, IImageRetriever imageRetriever, IImageSaver imageSaver, MockFileSystem fileSystem, IDirectoryHelper directoryHelper) + private static ImagePage BuildImagePageReturningSkippedImage() + { + var tagLocator = Substitute.For(); + tagLocator.InnerTextAsync().Returns(Task.FromResult("BadCategory")); + tagLocator.GetAttributeAsync("original-title").Returns(Task.FromResult("ExcludedCategory")); + + var tagsLocator = Substitute.For(); + tagsLocator.AllAsync().Returns(Task.FromResult>([tagLocator,])); + + var page = Substitute.For(); + page.Locator(".tagname", Arg.Any()).Returns(tagsLocator); + + var playwrightService = Substitute.For(); + playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + + var scrapeConfiguration = new ScrapeConfigurationBuilder().Build(); + + return new ImagePage(playwrightService, scrapeConfiguration, new() { Tags = ["ExcludedCategory",], }, new()); + } + + private ImagePageService BuildService( + ImagePage imagePage, + IFileDetailRepository fileDetailRepository, + IDelayStrategy delayStrategy, + IImageRetriever imageRetriever, + IImageSaver imageSaver, + MockFileSystem fileSystem, + IDirectoryHelper directoryHelper, + IScrapedTagRepository? scrapedTagRepository = null, + IImageDimensionReader? imageDimensionReader = null) { var contextFactory = Substitute.For>(); contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); var fileClassificationService = new FileClassificationService(contextFactory); var scrapeConfiguration = new ScrapeConfigurationBuilder().Build(); - return new ImagePageService(imagePage, fileDetailRepository, fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), directoryHelper, new(), delayStrategy, imageRetriever, imageSaver, fileSystem); + return new ImagePageService( + imagePage, + fileDetailRepository, + fileClassificationService, + scrapeConfiguration, + System.TimeProvider.System, + new LoggerConfiguration().CreateLogger(), + directoryHelper, + new(), + delayStrategy, + imageRetriever, + imageSaver, + fileSystem, + scrapedTagRepository ?? Substitute.For(), + imageDimensionReader ?? BuildDefaultImageDimensionReader()); + } + + private static IImageDimensionReader BuildDefaultImageDimensionReader() + { + var imageDimensionReader = Substitute.For(); + imageDimensionReader.Read(Arg.Any(), Arg.Any()) + .Returns(Result.Failure(ScrapeErrorFactory.CreateImageDimensionReadFailed("unset", "dimension reader not under test"))); + + return imageDimensionReader; + } + + private static IImageSaver BuildSucceedingImageSaver() + { + var imageSaver = Substitute.For(); + imageSaver.SaveAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result.Success(global::AStar.Dev.FunctionalParadigm.Unit.Value))); + + return imageSaver; } [Fact] @@ -77,7 +137,7 @@ public async Task when_the_file_already_exists_then_the_delay_strategy_receives_ fileDetailRepository.ExistsAsync(Arg.Any()).Returns(true); var delayStrategy = Substitute.For(); delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); - var imagePage = BuildImagePageReturningImage(ImageUrl); + var imagePage = BuildImagePageReturningScrapedImage(ImageUrl); var sut = BuildService(imagePage, fileDetailRepository, delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For()); @@ -93,17 +153,16 @@ public async Task when_the_file_does_not_already_exist_then_the_delay_strategy_r fileDetailRepository.ExistsAsync(Arg.Any()).Returns(false); var delayStrategy = Substitute.For(); delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); - var imagePage = BuildImagePageReturningImage(ImageUrl); + var imagePage = BuildImagePageReturningScrapedImage(ImageUrl); var directoryHelper = Substitute.For(); directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); var imageRetriever = Substitute.For(); imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result.Success([1, 2, 3,]))); - var imageSaver = Substitute.For(); var fileSystem = new MockFileSystem(); fileSystem.Directory.CreateDirectory("/save/dir"); fileSystem.File.WriteAllBytes("/save/dir/12345.data", [1, 2, 3,]); - var sut = BuildService(imagePage, fileDetailRepository, delayStrategy, imageRetriever, imageSaver, fileSystem, directoryHelper); + var sut = BuildService(imagePage, fileDetailRepository, delayStrategy, imageRetriever, BuildSucceedingImageSaver(), fileSystem, directoryHelper); await sut.GetTheImagePagesAsync([Link,], CategoryId, CategoryName, TestContext.Current.CancellationToken); @@ -111,25 +170,25 @@ public async Task when_the_file_does_not_already_exist_then_the_delay_strategy_r } [Fact] - public async Task when_the_image_retriever_always_fails_then_get_the_image_pages_async_retries_exactly_once_before_throwing() + public async Task when_the_image_retriever_always_fails_then_get_the_image_pages_async_retries_exactly_once_before_returning_a_failure() { var fileDetailRepository = Substitute.For(); fileDetailRepository.ExistsAsync(Arg.Any()).Returns(false); var delayStrategy = Substitute.For(); delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); - var imagePage = BuildImagePageReturningImage(ImageUrl); + var imagePage = BuildImagePageReturningScrapedImage(ImageUrl); var directoryHelper = Substitute.For(); directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); var imageRetriever = Substitute.For(); imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()) .Returns(Task.FromResult(Result.Failure(ScrapeErrorFactory.CreateImageDownloadFailed(ImageUrl, "download failed")))); - var imageSaver = Substitute.For(); var fileSystem = new MockFileSystem(); - var sut = BuildService(imagePage, fileDetailRepository, delayStrategy, imageRetriever, imageSaver, fileSystem, directoryHelper); + var sut = BuildService(imagePage, fileDetailRepository, delayStrategy, imageRetriever, Substitute.For(), fileSystem, directoryHelper); - await Should.ThrowAsync(() => sut.GetTheImagePagesAsync([Link,], CategoryId, CategoryName, TestContext.Current.CancellationToken)); + var exception = await Record.ExceptionAsync(() => sut.GetTheImagePagesAsync([Link,], CategoryId, CategoryName, TestContext.Current.CancellationToken)); + exception.ShouldBeNull(); await imageRetriever.Received(2).GetImageAsync(Arg.Any(), Arg.Any()); } @@ -140,19 +199,243 @@ public async Task when_the_image_retriever_always_fails_then_the_delay_strategy_ fileDetailRepository.ExistsAsync(Arg.Any()).Returns(false); var delayStrategy = Substitute.For(); delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); - var imagePage = BuildImagePageReturningImage(ImageUrl); + var imagePage = BuildImagePageReturningScrapedImage(ImageUrl); var directoryHelper = Substitute.For(); directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); var imageRetriever = Substitute.For(); imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()) .Returns(Task.FromResult(Result.Failure(ScrapeErrorFactory.CreateImageDownloadFailed(ImageUrl, "download failed")))); - var imageSaver = Substitute.For(); var fileSystem = new MockFileSystem(); - var sut = BuildService(imagePage, fileDetailRepository, delayStrategy, imageRetriever, imageSaver, fileSystem, directoryHelper); + var sut = BuildService(imagePage, fileDetailRepository, delayStrategy, imageRetriever, Substitute.For(), fileSystem, directoryHelper); - await Should.ThrowAsync(() => sut.GetTheImagePagesAsync([Link,], CategoryId, CategoryName, TestContext.Current.CancellationToken)); + await sut.GetTheImagePagesAsync([Link,], CategoryId, CategoryName, TestContext.Current.CancellationToken); await delayStrategy.Received(1).DelayAsync(DelayKind.Retry, Arg.Any()); } + + [Fact] + public async Task when_the_image_retriever_always_fails_then_the_final_result_is_an_image_download_failed_error() + { + var fileDetailRepository = Substitute.For(); + fileDetailRepository.ExistsAsync(Arg.Any()).Returns(false); + var imagePage = BuildImagePageReturningScrapedImage(ImageUrl); + var directoryHelper = Substitute.For(); + directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); + var imageRetriever = Substitute.For(); + imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(Result.Failure(ScrapeErrorFactory.CreateImageDownloadFailed(ImageUrl, "download failed")))); + + var sut = BuildService(imagePage, fileDetailRepository, new NoOpDelayStrategy(), imageRetriever, Substitute.For(), new MockFileSystem(), directoryHelper); + + var result = await sut.GetTheImagePagesAsync([Link,], CategoryId, CategoryName, TestContext.Current.CancellationToken); + + var error = result.ShouldBeOfType>().Error.ShouldBeOfType(); + error.ImageUrl.ShouldBe(ImageUrl); + } + + [Fact] + public async Task when_the_image_retriever_fails_once_then_succeeds_then_the_final_result_is_ok() + { + var fileDetailRepository = Substitute.For(); + fileDetailRepository.ExistsAsync(Arg.Any()).Returns(false); + var imagePage = BuildImagePageReturningScrapedImage(ImageUrl); + var directoryHelper = Substitute.For(); + directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); + var attempts = 0; + var imageRetriever = Substitute.For(); + imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()) + .Returns(_ => + { + attempts++; + + return Task.FromResult(attempts == 1 + ? Result.Failure(ScrapeErrorFactory.CreateImageDownloadFailed(ImageUrl, "download failed")) + : Result.Success([1, 2, 3,])); + }); + var fileSystem = new MockFileSystem(); + fileSystem.Directory.CreateDirectory("/save/dir"); + + var sut = BuildService(imagePage, fileDetailRepository, new NoOpDelayStrategy(), imageRetriever, BuildSucceedingImageSaver(), fileSystem, directoryHelper); + + var result = await sut.GetTheImagePagesAsync([Link,], CategoryId, CategoryName, TestContext.Current.CancellationToken); + + result.ShouldBeOfType>(); + } + + [Fact] + public async Task when_processing_a_scraped_image_then_the_scraped_tag_repository_is_saved_to_exactly_once() + { + var imagePage = BuildImagePageReturningScrapedImage(ImageUrl); + var directoryHelper = Substitute.For(); + directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); + var imageRetriever = Substitute.For(); + imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result.Success([1, 2, 3,]))); + var fileSystem = new MockFileSystem(); + fileSystem.Directory.CreateDirectory("/save/dir"); + var scrapedTagRepository = Substitute.For(); + + var sut = BuildService(imagePage, Substitute.For(), new NoOpDelayStrategy(), imageRetriever, BuildSucceedingImageSaver(), fileSystem, directoryHelper, scrapedTagRepository); + + await sut.ProcessImagePageAsync(Link, CategoryName, new PageClassificationData([], null, []), TestContext.Current.CancellationToken); + + await scrapedTagRepository.Received(1).SaveAsync(Arg.Any>()); + } + + [Fact] + public async Task when_processing_a_skipped_image_then_the_scraped_tag_repository_is_saved_to_exactly_once() + { + var imagePage = BuildImagePageReturningSkippedImage(); + var scrapedTagRepository = Substitute.For(); + + var sut = BuildService(imagePage, Substitute.For(), new NoOpDelayStrategy(), Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For(), scrapedTagRepository); + + await sut.ProcessImagePageAsync(Link, CategoryName, new PageClassificationData([], null, []), TestContext.Current.CancellationToken); + + await scrapedTagRepository.Received(1).SaveAsync(Arg.Any>()); + } + + [Fact] + public async Task when_processing_a_skipped_image_then_the_image_retriever_is_never_called() + { + var imagePage = BuildImagePageReturningSkippedImage(); + var imageRetriever = Substitute.For(); + + var sut = BuildService(imagePage, Substitute.For(), new NoOpDelayStrategy(), imageRetriever, Substitute.For(), new MockFileSystem(), Substitute.For()); + + await sut.ProcessImagePageAsync(Link, CategoryName, new PageClassificationData([], null, []), TestContext.Current.CancellationToken); + + await imageRetriever.DidNotReceive().GetImageAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task when_processing_a_skipped_image_then_the_result_is_ok() + { + var imagePage = BuildImagePageReturningSkippedImage(); + + var sut = BuildService(imagePage, Substitute.For(), new NoOpDelayStrategy(), Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For()); + + var result = await sut.ProcessImagePageAsync(Link, CategoryName, new PageClassificationData([], null, []), TestContext.Current.CancellationToken); + + result.ShouldBeOfType>(); + } + + [Fact] + public async Task when_processing_a_scraped_image_then_the_file_detail_repository_receives_exactly_one_add_call() + { + var imagePage = BuildImagePageReturningScrapedImage(ImageUrl); + var directoryHelper = Substitute.For(); + directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); + var imageRetriever = Substitute.For(); + imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result.Success([1, 2, 3,]))); + var fileSystem = new MockFileSystem(); + fileSystem.Directory.CreateDirectory("/save/dir"); + var fileDetailRepository = Substitute.For(); + + var sut = BuildService(imagePage, fileDetailRepository, new NoOpDelayStrategy(), imageRetriever, BuildSucceedingImageSaver(), fileSystem, directoryHelper); + + await sut.ProcessImagePageAsync(Link, CategoryName, new PageClassificationData([], null, []), TestContext.Current.CancellationToken); + + await fileDetailRepository.Received(1).AddAsync(Arg.Any()); + } + + [Fact] + public async Task when_the_image_dimension_reader_returns_dimensions_then_the_persisted_file_detail_carries_them() + { + var imagePage = BuildImagePageReturningScrapedImage(ImageUrl); + var directoryHelper = Substitute.For(); + directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); + var imageRetriever = Substitute.For(); + imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result.Success([1, 2, 3,]))); + var fileSystem = new MockFileSystem(); + fileSystem.Directory.CreateDirectory("/save/dir"); + var fileDetailRepository = Substitute.For(); + var imageDimensionReader = Substitute.For(); + imageDimensionReader.Read(Arg.Any(), Arg.Any()).Returns(Result.Success(new ImageDimensions(40, 20))); + + var sut = BuildService(imagePage, fileDetailRepository, new NoOpDelayStrategy(), imageRetriever, BuildSucceedingImageSaver(), fileSystem, directoryHelper, imageDimensionReader: imageDimensionReader); + + await sut.ProcessImagePageAsync(Link, CategoryName, new PageClassificationData([], null, []), TestContext.Current.CancellationToken); + + await fileDetailRepository.Received(1).AddAsync(Arg.Is(detail => detail.Width == 40 && detail.Height == 20)); + } + + [Fact] + public async Task when_the_image_dimension_reader_fails_then_the_file_detail_is_still_persisted() + { + var imagePage = BuildImagePageReturningScrapedImage(ImageUrl); + var directoryHelper = Substitute.For(); + directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); + var imageRetriever = Substitute.For(); + imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result.Success([1, 2, 3,]))); + var fileSystem = new MockFileSystem(); + fileSystem.Directory.CreateDirectory("/save/dir"); + var fileDetailRepository = Substitute.For(); + var imageDimensionReader = Substitute.For(); + imageDimensionReader.Read(Arg.Any(), Arg.Any()).Returns(Result.Failure(ScrapeErrorFactory.CreateImageDimensionReadFailed("/save/dir/12345.data", "invalid image"))); + + var sut = BuildService(imagePage, fileDetailRepository, new NoOpDelayStrategy(), imageRetriever, BuildSucceedingImageSaver(), fileSystem, directoryHelper, imageDimensionReader: imageDimensionReader); + + var result = await sut.ProcessImagePageAsync(Link, CategoryName, new PageClassificationData([], null, []), TestContext.Current.CancellationToken); + + result.ShouldBeOfType>(); + await fileDetailRepository.Received(1).AddAsync(Arg.Any()); + } + + [Fact] + public async Task when_the_scraped_file_is_not_an_image_extension_then_the_dimension_reader_is_still_invoked() + { + var imagePage = BuildImagePageReturningScrapedImage(ImageUrl); + var directoryHelper = Substitute.For(); + directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); + var imageRetriever = Substitute.For(); + imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result.Success([1, 2, 3,]))); + var fileSystem = new MockFileSystem(); + fileSystem.Directory.CreateDirectory("/save/dir"); + var imageDimensionReader = Substitute.For(); + imageDimensionReader.Read(Arg.Any(), Arg.Any()).Returns(Result.Failure(ScrapeErrorFactory.CreateImageDimensionReadFailed("/save/dir/12345.data", "dimension reader not under test"))); + + var sut = BuildService(imagePage, Substitute.For(), new NoOpDelayStrategy(), imageRetriever, BuildSucceedingImageSaver(), fileSystem, directoryHelper, imageDimensionReader: imageDimensionReader); + + await sut.ProcessImagePageAsync(Link, CategoryName, new PageClassificationData([], null, []), TestContext.Current.CancellationToken); + + imageDimensionReader.Received(1).Read(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task when_the_image_saver_fails_then_an_image_save_failed_error_is_returned() + { + var imagePage = BuildImagePageReturningScrapedImage(ImageUrl); + var directoryHelper = Substitute.For(); + directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); + var imageRetriever = Substitute.For(); + imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result.Success([1, 2, 3,]))); + var imageSaver = Substitute.For(); + imageSaver.SaveAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result.Failure(ScrapeErrorFactory.CreateImageSaveFailed("/save/dir/12345.data", "disk full")))); + + var sut = BuildService(imagePage, Substitute.For(), new NoOpDelayStrategy(), imageRetriever, imageSaver, new MockFileSystem(), directoryHelper); + + var result = await sut.ProcessImagePageAsync(Link, CategoryName, new PageClassificationData([], null, []), TestContext.Current.CancellationToken); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_the_image_saver_fails_then_the_file_detail_repository_is_never_called() + { + var imagePage = BuildImagePageReturningScrapedImage(ImageUrl); + var directoryHelper = Substitute.For(); + directoryHelper.CreateDirectoryIfRequired(Arg.Any>()).Returns(new DirectoryName("/save/dir")); + var imageRetriever = Substitute.For(); + imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result.Success([1, 2, 3,]))); + var imageSaver = Substitute.For(); + imageSaver.SaveAsync(Arg.Any(), Arg.Any()).Returns(Task.FromResult(Result.Failure(ScrapeErrorFactory.CreateImageSaveFailed("/save/dir/12345.data", "disk full")))); + var fileDetailRepository = Substitute.For(); + + var sut = BuildService(imagePage, fileDetailRepository, new NoOpDelayStrategy(), imageRetriever, imageSaver, new MockFileSystem(), directoryHelper); + + await sut.ProcessImagePageAsync(Link, CategoryName, new PageClassificationData([], null, []), TestContext.Current.CancellationToken); + + await fileDetailRepository.DidNotReceive().AddAsync(Arg.Any()); + } } diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImportExportService.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImportExportService.cs index 9d9b864..6810765 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImportExportService.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenAnImportExportService.cs @@ -1,5 +1,6 @@ using AStar.Dev.FunctionalParadigm; using AStar.Dev.Infrastructure.AppDb.Entities; +using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Services; using Microsoft.Extensions.Time.Testing; using Serilog; @@ -44,6 +45,7 @@ public sealed class GivenAnImportExportService { "id": 2, "name": "Test Normal", + "level": 2, "isFamous": false, "includeInSearch": true, "keywords": [ @@ -56,6 +58,14 @@ public sealed class GivenAnImportExportService ] """; + private const string ThreeInvalidRowsClassificationsJson = """ + [ + { "id": 1, "name": "", "level": 3, "isFamous": false, "includeInSearch": true, "keywords": [] }, + { "id": 2, "name": "Valid Name", "level": 0, "isFamous": false, "includeInSearch": true, "keywords": [] }, + { "id": 3, "name": "Another Valid", "level": 9, "isFamous": false, "includeInSearch": true, "keywords": [] } + ] + """; + private const string ValidScrapeConfigJson = """ { "connectionStrings": { "sqlite": "Data Source=test.db" }, @@ -115,7 +125,13 @@ public GivenAnImportExportService() [Fact] public void when_importing_and_file_does_not_exist_then_failure_result_is_returned() => sut.ImportFileClassificationsFromFile() - .ShouldBeOfType Categories, List Keywords), string>>(); + .ShouldBeOfType Categories, List Keywords), ScrapeError>>(); + + [Fact] + public void when_importing_and_file_does_not_exist_then_the_error_is_an_import_failed_error() => + sut.ImportFileClassificationsFromFile() + .ShouldBeOfType Categories, List Keywords), ScrapeError>>() + .Error.ShouldBeOfType(); [Fact] public void when_importing_and_file_does_not_exist_then_logger_receives_error_call() @@ -132,7 +148,7 @@ public void when_importing_and_file_contains_null_json_then_failure_result_is_re mockFileSystem.File.WriteAllText(ApplicationMetadata.FileClassificationsExportFilePath, "null"); sut.ImportFileClassificationsFromFile() - .ShouldBeOfType Categories, List Keywords), string>>(); + .ShouldBeOfType Categories, List Keywords), ScrapeError>>(); } [Fact] @@ -152,7 +168,7 @@ public void when_importing_valid_classifications_then_result_is_ok() SetupValidImportFile(); sut.ImportFileClassificationsFromFile() - .ShouldBeOfType Categories, List Keywords), string>>(); + .ShouldBeOfType Categories, List Keywords), ScrapeError>>(); } [Fact] @@ -161,7 +177,7 @@ public void when_importing_valid_classifications_then_correct_category_count_is_ SetupValidImportFile(); sut.ImportFileClassificationsFromFile() - .ShouldBeOfType Categories, List Keywords), string>>() + .ShouldBeOfType Categories, List Keywords), ScrapeError>>() .Value.Categories.Count.ShouldBe(2); } @@ -171,7 +187,7 @@ public void when_importing_valid_classifications_then_correct_keyword_count_is_r SetupValidImportFile(); sut.ImportFileClassificationsFromFile() - .ShouldBeOfType Categories, List Keywords), string>>() + .ShouldBeOfType Categories, List Keywords), ScrapeError>>() .Value.Keywords.Count.ShouldBe(2); } @@ -181,7 +197,7 @@ public void when_importing_valid_classifications_then_celebrity_classification_n SetupValidImportFile(); sut.ImportFileClassificationsFromFile() - .ShouldBeOfType Categories, List Keywords), string>>() + .ShouldBeOfType Categories, List Keywords), ScrapeError>>() .Value.Categories[0].Name.ShouldBe(CelebrityClassificationName); } @@ -191,7 +207,7 @@ public void when_importing_valid_classifications_then_normal_classification_name SetupValidImportFile(); sut.ImportFileClassificationsFromFile() - .ShouldBeOfType Categories, List Keywords), string>>() + .ShouldBeOfType Categories, List Keywords), ScrapeError>>() .Value.Categories[1].Name.ShouldBe(NormalClassificationName); } @@ -201,7 +217,7 @@ public void when_importing_valid_classifications_then_level_is_mapped() SetupValidImportFile(); sut.ImportFileClassificationsFromFile() - .ShouldBeOfType Categories, List Keywords), string>>() + .ShouldBeOfType Categories, List Keywords), ScrapeError>>() .Value.Categories[0].Level.ShouldBe(3); } @@ -211,10 +227,49 @@ public void when_importing_valid_classifications_then_keyword_is_linked_to_its_c SetupValidImportFile(); sut.ImportFileClassificationsFromFile() - .ShouldBeOfType Categories, List Keywords), string>>() + .ShouldBeOfType Categories, List Keywords), ScrapeError>>() .Value.Keywords[0].CategoryId.ShouldBe(1); } + [Fact] + public void when_importing_classifications_with_three_invalid_rows_then_the_result_is_a_validation_failed_error() + { + mockFileSystem.Directory.CreateDirectory(scrapperDirectory); + mockFileSystem.File.WriteAllText(ApplicationMetadata.FileClassificationsExportFilePath, ThreeInvalidRowsClassificationsJson); + + sut.ImportFileClassificationsFromFile() + .ShouldBeOfType Categories, List Keywords), ScrapeError>>() + .Error.ShouldBeOfType(); + } + + [Fact] + public void when_importing_classifications_with_three_invalid_rows_then_all_three_validation_errors_are_reported() + { + mockFileSystem.Directory.CreateDirectory(scrapperDirectory); + mockFileSystem.File.WriteAllText(ApplicationMetadata.FileClassificationsExportFilePath, ThreeInvalidRowsClassificationsJson); + + var errors = sut.ImportFileClassificationsFromFile() + .ShouldBeOfType Categories, List Keywords), ScrapeError>>() + .Error.ShouldBeOfType().Errors; + + errors.Count.ShouldBe(3); + } + + [Fact] + public void when_importing_classifications_with_three_invalid_rows_then_the_errors_are_reported_in_row_order() + { + mockFileSystem.Directory.CreateDirectory(scrapperDirectory); + mockFileSystem.File.WriteAllText(ApplicationMetadata.FileClassificationsExportFilePath, ThreeInvalidRowsClassificationsJson); + + var errors = sut.ImportFileClassificationsFromFile() + .ShouldBeOfType Categories, List Keywords), ScrapeError>>() + .Error.ShouldBeOfType().Errors; + + errors[0].Property.ShouldBe("Categories[0].Name"); + errors[1].Property.ShouldBe("Categories[1].Level"); + errors[2].Property.ShouldBe("Categories[2].Level"); + } + [Fact] public void when_exporting_and_reimporting_a_classification_then_level_survives_the_round_trip() { @@ -224,7 +279,7 @@ public void when_exporting_and_reimporting_a_classification_then_level_survives_ sut.ExportFileClassificationsToFile(classifications); sut.ImportFileClassificationsFromFile() - .ShouldBeOfType Categories, List Keywords), string>>() + .ShouldBeOfType Categories, List Keywords), ScrapeError>>() .Value.Categories[0].Level.ShouldBe(3); } @@ -237,7 +292,7 @@ public void when_exporting_and_reimporting_a_classification_then_keyword_survive sut.ExportFileClassificationsToFile(classifications); sut.ImportFileClassificationsFromFile() - .ShouldBeOfType Categories, List Keywords), string>>() + .ShouldBeOfType Categories, List Keywords), ScrapeError>>() .Value.Keywords.ShouldContain(k => k.Keyword == "celebrity-keyword"); } @@ -290,7 +345,13 @@ public void when_file_system_throws_during_export_then_logger_receives_error_cal [Fact] public void when_importing_scrape_config_and_file_does_not_exist_then_failure_result_is_returned() => sut.ImportScrapeConfigurationFromFile() - .ShouldBeOfType>(); + .ShouldBeOfType>(); + + [Fact] + public void when_importing_scrape_config_and_file_does_not_exist_then_the_error_is_an_import_failed_error() => + sut.ImportScrapeConfigurationFromFile() + .ShouldBeOfType>() + .Error.ShouldBeOfType(); [Fact] public void when_importing_scrape_config_and_file_does_not_exist_then_logger_receives_error_call() @@ -307,7 +368,7 @@ public void when_importing_scrape_config_and_file_contains_null_json_then_failur mockFileSystem.File.WriteAllText(ApplicationMetadata.ScrapeConfigurationExportFilePath, "null"); sut.ImportScrapeConfigurationFromFile() - .ShouldBeOfType>(); + .ShouldBeOfType>(); } [Fact] @@ -327,7 +388,7 @@ public void when_importing_valid_scrape_config_then_result_is_ok() SetupValidScrapeConfigImportFile(); sut.ImportScrapeConfigurationFromFile() - .ShouldBeOfType>(); + .ShouldBeOfType>(); } [Fact] @@ -336,7 +397,7 @@ public void when_importing_valid_scrape_config_then_correct_connection_string_is SetupValidScrapeConfigImportFile(); sut.ImportScrapeConfigurationFromFile() - .ShouldBeOfType>() + .ShouldBeOfType>() .Value.ConnectionStrings.Sqlite.ShouldBe("Data Source=test.db"); } @@ -346,7 +407,7 @@ public void when_importing_valid_scrape_config_then_password_field_is_preserved_ SetupValidScrapeConfigImportFile(); sut.ImportScrapeConfigurationFromFile() - .ShouldBeOfType>() + .ShouldBeOfType>() .Value.UserConfiguration.Password.ShouldBe(ApplicationMetadata.Redacted); } @@ -412,7 +473,13 @@ public void when_file_system_throws_during_scrape_config_export_then_logger_rece [Fact] public void when_importing_tags_and_file_does_not_exist_then_failure_result_is_returned() => sut.ImportScrapedTagsFromFile() - .ShouldBeOfType, string>>(); + .ShouldBeOfType, ScrapeError>>(); + + [Fact] + public void when_importing_tags_and_file_does_not_exist_then_the_error_is_an_import_failed_error() => + sut.ImportScrapedTagsFromFile() + .ShouldBeOfType, ScrapeError>>() + .Error.ShouldBeOfType(); [Fact] public void when_importing_tags_and_file_does_not_exist_then_logger_receives_error_call() @@ -429,7 +496,7 @@ public void when_importing_tags_and_file_contains_null_json_then_failure_result_ mockFileSystem.File.WriteAllText(ApplicationMetadata.ScrapedTagsExportFilePath, "null"); sut.ImportScrapedTagsFromFile() - .ShouldBeOfType, string>>(); + .ShouldBeOfType, ScrapeError>>(); } [Fact] @@ -449,7 +516,7 @@ public void when_importing_valid_tags_then_result_is_ok() SetupValidTagsImportFile(); sut.ImportScrapedTagsFromFile() - .ShouldBeOfType, string>>(); + .ShouldBeOfType, ScrapeError>>(); } [Fact] @@ -458,7 +525,7 @@ public void when_importing_valid_tags_then_correct_count_is_returned() SetupValidTagsImportFile(); sut.ImportScrapedTagsFromFile() - .ShouldBeOfType, string>>() + .ShouldBeOfType, ScrapeError>>() .Value.Count.ShouldBe(2); } @@ -468,7 +535,7 @@ public void when_importing_valid_tags_then_first_tag_value_is_mapped() SetupValidTagsImportFile(); sut.ImportScrapedTagsFromFile() - .ShouldBeOfType, string>>() + .ShouldBeOfType, ScrapeError>>() .Value[0].Value.ShouldBe(ActionTagValue); } @@ -478,7 +545,7 @@ public void when_importing_valid_tags_then_first_tag_category_is_mapped() SetupValidTagsImportFile(); sut.ImportScrapedTagsFromFile() - .ShouldBeOfType, string>>() + .ShouldBeOfType, ScrapeError>>() .Value[0].Category.ShouldBe(GenreCategory); } @@ -488,7 +555,7 @@ public void when_importing_valid_tags_then_first_tag_include_in_search_is_mapped SetupValidTagsImportFile(); sut.ImportScrapedTagsFromFile() - .ShouldBeOfType, string>>() + .ShouldBeOfType, ScrapeError>>() .Value[0].IncludeInSearch.ShouldBeTrue(); } @@ -498,7 +565,7 @@ public void when_importing_valid_tags_then_second_tag_value_is_mapped() SetupValidTagsImportFile(); sut.ImportScrapedTagsFromFile() - .ShouldBeOfType, string>>() + .ShouldBeOfType, ScrapeError>>() .Value[1].Value.ShouldBe(ComedyTagValue); } @@ -508,7 +575,7 @@ public void when_importing_valid_tags_then_second_tag_category_is_mapped() SetupValidTagsImportFile(); sut.ImportScrapedTagsFromFile() - .ShouldBeOfType, string>>() + .ShouldBeOfType, ScrapeError>>() .Value[1].Category.ShouldBe(GenreCategory); } @@ -518,7 +585,7 @@ public void when_importing_valid_tags_then_second_tag_include_in_search_is_mappe SetupValidTagsImportFile(); sut.ImportScrapedTagsFromFile() - .ShouldBeOfType, string>>() + .ShouldBeOfType, ScrapeError>>() .Value[1].IncludeInSearch.ShouldBeFalse(); } diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenTheClassificationMatcher.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenTheClassificationMatcher.cs new file mode 100644 index 0000000..7484bc6 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Services/GivenTheClassificationMatcher.cs @@ -0,0 +1,87 @@ +using AStar.Dev.Infrastructure.AppDb.Entities; +using AStar.Dev.Wallpaper.Scrapper.Services; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Services; + +public sealed class GivenTheClassificationMatcher +{ + private static FileDetailEntity BuildFileDetail(string fileName) + => new() { FileName = new FileName(fileName), DirectoryName = new DirectoryName("/tmp"), }; + + [Fact] + public void when_there_are_no_searchable_classifications_and_no_category_classification_then_the_result_is_empty() + { + var pageData = new PageClassificationData([], null, []); + + var matched = ClassificationMatcher.Match(pageData, BuildFileDetail("test.jpg")); + + matched.ShouldBeEmpty(); + } + + [Fact] + public void when_a_searchable_classification_keyword_matches_the_file_name_then_it_is_included() + { + var category = new FileClassificationCategoryEntity { Name = "Animals", Level = 3, }; + var pageData = new PageClassificationData([(category, ["animals",])], null, []); + + var matched = ClassificationMatcher.Match(pageData, BuildFileDetail("animals-photo.jpg")); + + matched.ShouldContain(category); + } + + [Fact] + public void when_a_searchable_classification_keyword_does_not_match_the_file_name_then_it_is_not_included() + { + var category = new FileClassificationCategoryEntity { Name = "Animals", Level = 3, }; + var pageData = new PageClassificationData([(category, ["animals",])], null, []); + + var matched = ClassificationMatcher.Match(pageData, BuildFileDetail("cars-photo.jpg")); + + matched.ShouldNotContain(category); + } + + [Fact] + public void when_a_category_classification_is_supplied_then_it_is_included() + { + var categoryClassification = new FileClassificationCategoryEntity { Name = "Cars", Level = 3, }; + var pageData = new PageClassificationData([], categoryClassification, []); + + var matched = ClassificationMatcher.Match(pageData, BuildFileDetail("test.jpg")); + + matched.ShouldContain(categoryClassification); + } + + [Fact] + public void when_no_category_classification_is_supplied_then_none_is_included_from_it() + { + var pageData = new PageClassificationData([], null, []); + + var matched = ClassificationMatcher.Match(pageData, BuildFileDetail("test.jpg")); + + matched.ShouldBeEmpty(); + } + + [Fact] + public void when_both_a_filename_match_and_a_category_classification_are_present_then_both_are_included() + { + var fileNameCategory = new FileClassificationCategoryEntity { Name = "Animals", Level = 3, }; + var categoryClassification = new FileClassificationCategoryEntity { Name = "Cars", Level = 3, }; + var pageData = new PageClassificationData([(fileNameCategory, ["animals",])], categoryClassification, []); + + var matched = ClassificationMatcher.Match(pageData, BuildFileDetail("animals-photo.jpg")); + + matched.Count.ShouldBe(2); + } + + [Fact] + public void when_multiple_searchable_classifications_match_the_file_name_then_all_are_included() + { + var animals = new FileClassificationCategoryEntity { Name = "Animals", Level = 3, }; + var outdoors = new FileClassificationCategoryEntity { Name = "Outdoors", Level = 3, }; + var pageData = new PageClassificationData([(animals, ["animals",]), (outdoors, ["outdoors",])], null, []); + + var matched = ClassificationMatcher.Match(pageData, BuildFileDetail("animals-outdoors-photo.jpg")); + + matched.Count.ShouldBe(2); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenAConfigurationSaver.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenAConfigurationSaver.cs new file mode 100644 index 0000000..1625a3e --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenAConfigurationSaver.cs @@ -0,0 +1,98 @@ +using NSubstitute.ExceptionExtensions; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Infrastructure.AppDb; +using AStar.Dev.Infrastructure.AppDb.Entities; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Support; +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Serilog; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Support; + +public sealed class GivenAConfigurationSaver : IAsyncLifetime +{ + private SqliteConnection connection = null!; + private DbContextOptions options = null!; + + public async ValueTask InitializeAsync() + { + connection = new SqliteConnection("Data Source=:memory:"); + await connection.OpenAsync(); + + options = new DbContextOptionsBuilder().UseSqlite(connection).Options; + + await using var seedContext = new AppDbContext(options); + await seedContext.Database.MigrateAsync(); + + seedContext.ScrapeConfiguration.Add(new ScrapeConfigurationEntity + { + ConnectionStrings = new ConnectionStringsEntity { Sqlite = "Data Source=test.db", }, + UserConfiguration = new UserConfigurationEntity { LoginEmailAddress = "user@example.com", Username = "user", Password = "password", SessionCookie = "cookie", }, + SearchConfiguration = new SearchConfigurationEntity { BaseUrl = new Uri("https://example.com"), }, + ScrapeDirectories = new ScrapeDirectoriesEntity { RootDirectory = "root-directory", }, + }); + await seedContext.SaveChangesAsync(); + } + + public async ValueTask DisposeAsync() => await connection.DisposeAsync(); + + [Fact] + public async Task when_saving_completes_successfully_then_the_result_is_ok() + { + var category = new Category { Id = "cat-1", Name = "Cars", LastKnownImageCount = 10, LastPageVisited = 2, TotalPages = 5, }; + var searchConfiguration = new SearchConfigurationBuilder { SearchCategories = [category,], }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); + var sut = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + + var result = await sut.SaveUpdatedConfigurationAsync(); + + result.ShouldBeOfType>(); + } + + [Fact] + public async Task when_saving_a_new_category_then_it_is_persisted() + { + var category = new Category { Id = "cat-new", Name = "Nature", LastKnownImageCount = 3, LastPageVisited = 1, TotalPages = 2, }; + var searchConfiguration = new SearchConfigurationBuilder { SearchCategories = [category,], }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); + var sut = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + + await sut.SaveUpdatedConfigurationAsync(); + + await using var verifyContext = new AppDbContext(options); + var saved = await verifyContext.SearchConfigurations.SelectMany(sc => sc.SearchCategories).SingleAsync(c => c.Id == "cat-new", TestContext.Current.CancellationToken); + saved.Name.ShouldBe("Nature"); + } + + [Fact] + public async Task when_the_context_factory_throws_then_a_configuration_save_failed_error_is_returned() + { + var scrapeConfiguration = new ScrapeConfigurationBuilder().Build(); + var contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).ThrowsAsync(new InvalidOperationException("db unavailable")); + var sut = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + + var result = await sut.SaveUpdatedConfigurationAsync(); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_the_context_factory_throws_then_no_exception_is_thrown() + { + var scrapeConfiguration = new ScrapeConfigurationBuilder().Build(); + var contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).ThrowsAsync(new InvalidOperationException("db unavailable")); + var sut = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + + var exception = await Record.ExceptionAsync(() => sut.SaveUpdatedConfigurationAsync()); + + exception.ShouldBeNull(); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenASqliteConnectionStringProvider.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenASqliteConnectionStringProvider.cs index 130774d..d7a4a19 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenASqliteConnectionStringProvider.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenASqliteConnectionStringProvider.cs @@ -9,7 +9,7 @@ public sealed class GivenASqliteConnectionStringProvider public void when_the_configuration_has_a_sqlite_connection_string_then_the_configured_value_is_used() { var configuration = new ConfigurationBuilder() - .AddInMemoryCollection(new Dictionary { ["ConnectionStrings:Sqlite"] = "Data Source=/configured/path/scrapper.db", }) + .AddInMemoryCollection(new Dictionary { ["ScrapeConfiguration:ConnectionStrings:Sqlite"] = "Data Source=/configured/path/scrapper.db", }) .Build(); SqliteConnectionStringProvider.Get(configuration).ShouldBe("Data Source=/configured/path/scrapper.db"); diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheCategoryMerger.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheCategoryMerger.cs new file mode 100644 index 0000000..1dc7654 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheCategoryMerger.cs @@ -0,0 +1,104 @@ +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Support; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Support; + +public sealed class GivenTheCategoryMerger +{ + private static Category BuildCategory(string id, string name, int lastKnownImageCount, int lastPageVisited, int totalPages) + => new() { Id = id, Name = name, LastKnownImageCount = lastKnownImageCount, LastPageVisited = lastPageVisited, TotalPages = totalPages, }; + + [Fact] + public void when_an_updated_category_matches_an_existing_category_then_it_appears_in_to_update() + { + (string Id, string Name, int LastKnownImageCount, int LastPageVisited, int TotalPages)[] existing = [("cat-1", "Cars", 1, 1, 1),]; + var updated = new[] { BuildCategory("cat-1", "Cars", 10, 2, 5), }; + + var (toUpdate, _) = CategoryMerger.MergeCategories(existing, updated); + + toUpdate.ShouldHaveSingleItem().Id.ShouldBe("cat-1"); + } + + [Fact] + public void when_an_updated_category_matches_an_existing_category_then_the_updated_counts_are_used() + { + (string Id, string Name, int LastKnownImageCount, int LastPageVisited, int TotalPages)[] existing = [("cat-1", "Cars", 1, 1, 1),]; + var updated = new[] { BuildCategory("cat-1", "Cars", 10, 2, 5), }; + + var (toUpdate, _) = CategoryMerger.MergeCategories(existing, updated); + + var item = toUpdate.ShouldHaveSingleItem(); + item.LastKnownImageCount.ShouldBe(10); + item.LastPageVisited.ShouldBe(2); + item.TotalPages.ShouldBe(5); + } + + [Fact] + public void when_an_updated_category_matches_an_existing_category_then_it_does_not_appear_in_to_add() + { + (string Id, string Name, int LastKnownImageCount, int LastPageVisited, int TotalPages)[] existing = [("cat-1", "Cars", 1, 1, 1),]; + var updated = new[] { BuildCategory("cat-1", "Cars", 10, 2, 5), }; + + var (_, toAdd) = CategoryMerger.MergeCategories(existing, updated); + + toAdd.ShouldBeEmpty(); + } + + [Fact] + public void when_an_updated_category_does_not_match_any_existing_category_then_it_appears_in_to_add() + { + var updated = new[] { BuildCategory("cat-new", "Nature", 3, 0, 2), }; + + var (_, toAdd) = CategoryMerger.MergeCategories([], updated); + + toAdd.ShouldHaveSingleItem().Id.ShouldBe("cat-new"); + } + + [Fact] + public void when_an_updated_category_does_not_match_any_existing_category_then_the_name_is_carried_to_add() + { + var updated = new[] { BuildCategory("cat-new", "Nature", 3, 0, 2), }; + + var (_, toAdd) = CategoryMerger.MergeCategories([], updated); + + toAdd.ShouldHaveSingleItem().Name.ShouldBe("Nature"); + } + + [Fact] + public void when_an_updated_category_does_not_match_any_existing_category_then_it_does_not_appear_in_to_update() + { + var updated = new[] { BuildCategory("cat-new", "Nature", 3, 0, 2), }; + + var (toUpdate, _) = CategoryMerger.MergeCategories([], updated); + + toUpdate.ShouldBeEmpty(); + } + + [Fact] + public void when_updated_contains_duplicate_ids_then_only_the_first_occurrence_is_merged() + { + var updated = new[] { BuildCategory("cat-1", "First", 1, 1, 1), BuildCategory("cat-1", "Second", 99, 99, 99), }; + + var (_, toAdd) = CategoryMerger.MergeCategories([], updated); + + toAdd.ShouldHaveSingleItem().Name.ShouldBe("First"); + } + + [Fact] + public void when_updated_is_empty_then_to_update_is_empty() + { + (string Id, string Name, int LastKnownImageCount, int LastPageVisited, int TotalPages)[] existing = [("cat-1", "Cars", 1, 1, 1),]; + + var (toUpdate, _) = CategoryMerger.MergeCategories(existing, []); + + toUpdate.ShouldBeEmpty(); + } + + [Fact] + public void when_updated_is_empty_then_to_add_is_empty() + { + var (_, toAdd) = CategoryMerger.MergeCategories([], []); + + toAdd.ShouldBeEmpty(); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheImageDimensionReader.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheImageDimensionReader.cs new file mode 100644 index 0000000..517c28c --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheImageDimensionReader.cs @@ -0,0 +1,51 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Support; +using SkiaSharp; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Support; + +public sealed class GivenTheImageDimensionReader +{ + private const string Path = "/save/dir/image.png"; + private readonly IImageDimensionReader sut = new ImageDimensionReader(); + + private static byte[] CreateValidPngBytes(int width, int height) + { + using var bitmap = new SKBitmap(width, height); + using var image = SKImage.FromBitmap(bitmap); + using var data = image.Encode(SKEncodedImageFormat.Png, 100); + + return data.ToArray(); + } + + [Fact] + public void when_the_bytes_are_a_valid_image_then_the_width_is_returned() + { + var bytes = CreateValidPngBytes(40, 20); + + sut.Read(bytes, Path).ShouldBeOfType>().Value.Width.ShouldBe(40); + } + + [Fact] + public void when_the_bytes_are_a_valid_image_then_the_height_is_returned() + { + var bytes = CreateValidPngBytes(40, 20); + + sut.Read(bytes, Path).ShouldBeOfType>().Value.Height.ShouldBe(20); + } + + [Fact] + public void when_the_bytes_are_not_a_valid_image_then_an_image_dimension_read_failed_error_is_returned() + { + byte[] invalidBytes = [1, 2, 3, 4,]; + + var error = sut.Read(invalidBytes, Path).ShouldBeOfType>().Error.ShouldBeOfType(); + + error.Path.ShouldBe(Path); + } + + [Fact] + public void when_the_bytes_are_empty_then_an_image_dimension_read_failed_error_is_returned() => + sut.Read([], Path).ShouldBeOfType>().Error.ShouldBeOfType(); +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheImageSaver.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheImageSaver.cs index 367a2c6..1bb4e61 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheImageSaver.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheImageSaver.cs @@ -1,4 +1,8 @@ +using NSubstitute.ExceptionExtensions; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; using AStar.Dev.Wallpaper.Scrapper.Support; +using System.IO.Abstractions; namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Support; @@ -19,6 +23,16 @@ public async Task when_the_image_is_empty_then_no_file_is_written() fileSystem.File.Exists("/save/dir/image.jpg").ShouldBeFalse(); } + [Fact] + public async Task when_the_image_is_empty_then_the_result_is_ok() + { + fileSystem.Directory.CreateDirectory("/save/dir"); + + var result = await sut.SaveAsync([], "/save/dir/image.jpg"); + + result.ShouldBeOfType>(); + } + [Fact] public async Task when_the_image_is_not_empty_then_the_file_is_written_with_the_supplied_bytes() { @@ -30,6 +44,17 @@ public async Task when_the_image_is_not_empty_then_the_file_is_written_with_the_ fileSystem.File.ReadAllBytes("/save/dir/image.jpg").ShouldBe(image); } + [Fact] + public async Task when_the_image_is_not_empty_then_the_result_is_ok() + { + fileSystem.Directory.CreateDirectory("/save/dir"); + byte[] image = [1, 2, 3,]; + + var result = await sut.SaveAsync(image, "/save/dir/image.jpg"); + + result.ShouldBeOfType>(); + } + [Fact] public async Task when_the_path_contains_a_colon_after_the_second_character_then_it_is_replaced_with_an_underscore() { @@ -51,4 +76,30 @@ public async Task when_the_path_contains_a_quote_then_it_is_replaced_with_a_sing fileSystem.File.Exists("/save/dir/file'name.jpg").ShouldBeTrue(); } + + [Fact] + public async Task when_the_file_system_throws_then_an_image_save_failed_error_is_returned() + { + var throwingFileSystem = Substitute.For(); + throwingFileSystem.File.WriteAllBytesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .ThrowsAsync(new IOException("disk full")); + var throwingSut = new ImageSaver(throwingFileSystem); + + var error = (await throwingSut.SaveAsync([1, 2,], "/save/dir/image.jpg")).ShouldBeOfType>().Error.ShouldBeOfType(); + + error.Path.ShouldBe("/save/dir/image.jpg"); + } + + [Fact] + public async Task when_the_file_system_throws_then_no_exception_is_thrown() + { + var throwingFileSystem = Substitute.For(); + throwingFileSystem.File.WriteAllBytesAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .ThrowsAsync(new IOException("disk full")); + var throwingSut = new ImageSaver(throwingFileSystem); + + var exception = await Record.ExceptionAsync(() => throwingSut.SaveAsync([1, 2,], "/save/dir/image.jpg")); + + exception.ShouldBeNull(); + } } diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowThatFails.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowThatFails.cs new file mode 100644 index 0000000..c9786d4 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowThatFails.cs @@ -0,0 +1,100 @@ +using NSubstitute.ExceptionExtensions; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Infrastructure.AppDb; +using AStar.Dev.Infrastructure.AppDb.Entities; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Pages; +using AStar.Dev.Wallpaper.Scrapper.Repositories; +using AStar.Dev.Wallpaper.Scrapper.Support; +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; +using AStar.Dev.Wallpaper.Scrapper.Workflows; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Playwright; +using Serilog; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Workflows; + +public sealed class GivenASearchWorkflowThatFails : IAsyncLifetime +{ + private const string SearchStringPrefix = "https://example.test/search/"; + private const string SearchStringSuffix = "/?page="; + private const string CategoryId = "cat-fails"; + + private SqliteConnection connection = null!; + private DbContextOptions options = null!; + + public async ValueTask InitializeAsync() + { + connection = new SqliteConnection("Data Source=:memory:"); + await connection.OpenAsync(); + + options = new DbContextOptionsBuilder().UseSqlite(connection).Options; + + await using var seedContext = new AppDbContext(options); + await seedContext.Database.MigrateAsync(); + + seedContext.ScrapeConfiguration.Add(new ScrapeConfigurationEntity + { + ConnectionStrings = new ConnectionStringsEntity { Sqlite = "Data Source=test.db", }, + UserConfiguration = new UserConfigurationEntity { LoginEmailAddress = "user@example.com", Username = "user", Password = "password", SessionCookie = "cookie", }, + SearchConfiguration = new SearchConfigurationEntity { BaseUrl = new Uri("https://example.com"), }, + ScrapeDirectories = new ScrapeDirectoriesEntity { RootDirectory = "root-directory", }, + }); + await seedContext.SaveChangesAsync(); + } + + public async ValueTask DisposeAsync() => await connection.DisposeAsync(); + + private SearchWorkflow BuildSut(IPage page, ILogger logger) + { + var playwrightService = Substitute.For(); + playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + + var category = new Category { Id = CategoryId, Name = "Fails Category", LastKnownImageCount = 999, TotalPages = 999, }; + var searchConfiguration = new SearchConfigurationBuilder + { + SearchCategories = [category,], + SearchString = $"{SearchStringPrefix}{category.Id}{SearchStringSuffix}", + SearchStringPrefix = SearchStringPrefix, + SearchStringSuffix = SearchStringSuffix, + }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + + var contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); + + var searchResultsPage = new SearchResultsPage(playwrightService); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new()); + var fileClassificationService = new FileClassificationService(contextFactory); + var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), new NoOpDelayStrategy(), Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For(), Substitute.For()); + + return new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), logger, new NoOpDelayStrategy(), System.TimeProvider.System); + } + + [Fact] + public async Task when_the_search_results_page_fails_to_load_then_run_async_returns_a_failure() + { + var page = Substitute.For(); + page.GotoAsync(Arg.Any(), Arg.Any()).ThrowsAsync(new PlaywrightException("navigation failed")); + var sut = BuildSut(page, Substitute.For()); + + var result = await sut.RunAsync(TestContext.Current.CancellationToken); + + result.ShouldBeOfType>().Error.ShouldBeOfType(); + } + + [Fact] + public async Task when_the_search_results_page_fails_to_load_then_the_logger_receives_exactly_one_error_call() + { + var page = Substitute.For(); + page.GotoAsync(Arg.Any(), Arg.Any()).ThrowsAsync(new PlaywrightException("navigation failed")); + var logger = Substitute.For(); + var sut = BuildSut(page, logger); + + await sut.RunAsync(TestContext.Current.CancellationToken); + + logger.Received(1).Error(Arg.Any(), Arg.Any()); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithADelayStrategy.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithADelayStrategy.cs index 70b33e1..0c29b03 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithADelayStrategy.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithADelayStrategy.cs @@ -85,17 +85,17 @@ public async Task when_a_category_is_up_to_date_then_the_delay_strategy_receives var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); var contextFactory = Substitute.For>(); - var searchResultsPage = new SearchResultsPage(playwrightService, new LoggerConfiguration().CreateLogger()); + var searchResultsPage = new SearchResultsPage(playwrightService); var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); - var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new(), Substitute.For()); + var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new()); var fileClassificationService = new FileClassificationService(contextFactory); var delayStrategy = Substitute.For(); delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); - var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem()); + var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For(), Substitute.For()); var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), delayStrategy, System.TimeProvider.System); - await sut.RunAsync(Substitute.For(), TestContext.Current.CancellationToken); + await sut.RunAsync(TestContext.Current.CancellationToken); await delayStrategy.Received(1).DelayAsync(DelayKind.CategoryUpToDate, Arg.Any()); } @@ -119,17 +119,17 @@ public async Task when_a_category_is_not_up_to_date_then_the_delay_strategy_rece var contextFactory = Substitute.For>(); contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); - var searchResultsPage = new SearchResultsPage(playwrightService, new LoggerConfiguration().CreateLogger()); + var searchResultsPage = new SearchResultsPage(playwrightService); var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); - var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new(), Substitute.For()); + var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new()); var fileClassificationService = new FileClassificationService(contextFactory); var delayStrategy = Substitute.For(); delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); - var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem()); + var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For(), Substitute.For()); var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), delayStrategy, System.TimeProvider.System); - await sut.RunAsync(Substitute.For(), TestContext.Current.CancellationToken); + await sut.RunAsync(TestContext.Current.CancellationToken); await delayStrategy.Received(2).DelayAsync(DelayKind.PageNavigation, Arg.Any()); } @@ -153,17 +153,17 @@ public async Task when_the_delay_strategy_is_a_zero_delay_substitute_then_run_as var contextFactory = Substitute.For>(); contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); - var searchResultsPage = new SearchResultsPage(playwrightService, new LoggerConfiguration().CreateLogger()); + var searchResultsPage = new SearchResultsPage(playwrightService); var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); - var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new(), Substitute.For()); + var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new()); var fileClassificationService = new FileClassificationService(contextFactory); var delayStrategy = new NoOpDelayStrategy(); - var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem()); + var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For(), Substitute.For()); var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), delayStrategy, System.TimeProvider.System); var stopwatch = Stopwatch.StartNew(); - await sut.RunAsync(Substitute.For(), TestContext.Current.CancellationToken); + await sut.RunAsync(TestContext.Current.CancellationToken); stopwatch.Stop(); stopwatch.Elapsed.ShouldBeLessThan(TimeSpan.FromSeconds(2)); diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithAMidListSearchString.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithAMidListSearchString.cs index 3efb8da..df30886 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithAMidListSearchString.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithAMidListSearchString.cs @@ -50,15 +50,15 @@ public async Task when_run_then_categories_before_the_matching_category_are_not_ var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); var contextFactory = Substitute.For>(); - var searchResultsPage = new SearchResultsPage(playwrightService, new LoggerConfiguration().CreateLogger()); + var searchResultsPage = new SearchResultsPage(playwrightService); var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); - var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new(), Substitute.For()); + var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new()); var fileClassificationService = new FileClassificationService(contextFactory); - var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), new NoOpDelayStrategy(), Substitute.For(), Substitute.For(), new MockFileSystem()); + var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), new NoOpDelayStrategy(), Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For(), Substitute.For()); var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), new NoOpDelayStrategy(), System.TimeProvider.System); - await sut.RunAsync(Substitute.For(), TestContext.Current.CancellationToken); + await sut.RunAsync(TestContext.Current.CancellationToken); await page.DidNotReceive().GotoAsync(Arg.Is(url => url.Contains(CategoryBeforeMatchId)), Arg.Any()); } diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithANonEmptySubDirectory.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithANonEmptySubDirectory.cs index 072b48a..f89add7 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithANonEmptySubDirectory.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithANonEmptySubDirectory.cs @@ -81,16 +81,16 @@ public async Task when_the_page_reports_a_non_empty_sub_directory_name_then_the_ var contextFactory = Substitute.For>(); contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); - var searchResultsPage = new SearchResultsPage(playwrightService, new LoggerConfiguration().CreateLogger()); + var searchResultsPage = new SearchResultsPage(playwrightService); var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); - var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new(), Substitute.For()); + var imagePage = new ImagePage(playwrightService, scrapeConfiguration, new(), new()); var fileClassificationService = new FileClassificationService(contextFactory); - var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), new NoOpDelayStrategy(), Substitute.For(), Substitute.For(), new MockFileSystem()); + var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), new NoOpDelayStrategy(), Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For(), Substitute.For()); var directoryHelper = Substitute.For(); var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, directoryHelper, Substitute.For(), new NoOpDelayStrategy(), System.TimeProvider.System); - await sut.RunAsync(Substitute.For(), TestContext.Current.CancellationToken); + await sut.RunAsync(TestContext.Current.CancellationToken); List expectedDirectoryPath = [Path.Combine(RootDirectory, BaseDirectory, ExpectedSubDirectoryName),]; diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASubscriptionsWorkflowCompiles.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASubscriptionsWorkflowCompiles.cs new file mode 100644 index 0000000..abe0057 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASubscriptionsWorkflowCompiles.cs @@ -0,0 +1,119 @@ +using NSubstitute.ExceptionExtensions; +using AStar.Dev.Infrastructure.AppDb; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Pages; +using AStar.Dev.Wallpaper.Scrapper.Repositories; +using AStar.Dev.Wallpaper.Scrapper.Support; +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; +using AStar.Dev.Wallpaper.Scrapper.Workflows; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Playwright; +using Serilog; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Workflows; + +public sealed class GivenASubscriptionsWorkflowCompiles +{ + [Fact] + public async Task when_run_against_the_result_based_page_contract_then_no_exception_is_thrown() + { + var headerLocator = Substitute.For(); + headerLocator.TextContentAsync().Returns(Task.FromResult("0 New Subscription Wallpapers")); + var linksLocator = Substitute.For(); + linksLocator.AllAsync().Returns(Task.FromResult>([])); + var page = Substitute.For(); + page.GetByText(Arg.Any(), Arg.Any()).Returns(headerLocator); + page.GetByRole(AriaRole.Link).Returns(linksLocator); + var response = Substitute.For(); + page.GotoAsync(Arg.Any()).Returns(Task.FromResult(response)); + + var playwrightService = Substitute.For(); + playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + + var searchConfiguration = new SearchConfigurationBuilder { SubscriptionsStartingPageNumber = 1, SubscriptionsTotalPages = 0, }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var subscriptionsImagesListPage = new SubscriptionsImagesListPage(playwrightService, searchConfiguration); + + var contextFactory = Substitute.For>(); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + var imagePageService = new ImagePageService( + new ImagePage(playwrightService, scrapeConfiguration, new(), new()), + Substitute.For(), + new FileClassificationService(contextFactory), + scrapeConfiguration, + System.TimeProvider.System, + new LoggerConfiguration().CreateLogger(), + Substitute.For(), + new(), + new NoOpDelayStrategy(), + Substitute.For(), + Substitute.For(), + new MockFileSystem(), + Substitute.For(), + Substitute.For()); + + var sut = new SubscriptionsWorkflow(subscriptionsImagesListPage, imagePageService, searchConfiguration, scrapeConfiguration.ScrapeDirectories, configurationSaver, new LoggerConfiguration().CreateLogger()); + + var exception = await Record.ExceptionAsync(() => sut.RunAsync(TestContext.Current.CancellationToken)); + + exception.ShouldBeNull(); + } + + [Fact] + public async Task when_the_image_page_service_reports_a_failure_then_run_async_throws_an_invalid_operation_exception() + { + await using var connection = new SqliteConnection("Data Source=:memory:"); + await connection.OpenAsync(TestContext.Current.CancellationToken); + var options = new DbContextOptionsBuilder().UseSqlite(connection).Options; + await using (var seedContext = new AppDbContext(options)) await seedContext.Database.MigrateAsync(TestContext.Current.CancellationToken); + + var headerLocator = Substitute.For(); + headerLocator.TextContentAsync().Returns(Task.FromResult("1 New Subscription Wallpapers")); + var linkLocator = Substitute.For(); + linkLocator.GetAttributeAsync("href").Returns(Task.FromResult("https://example.test/w/12345")); + var linksLocator = Substitute.For(); + linksLocator.AllAsync().Returns(Task.FromResult>([linkLocator,])); + var page = Substitute.For(); + page.GetByText(Arg.Any(), Arg.Any()).Returns(headerLocator); + page.GetByRole(AriaRole.Link).Returns(linksLocator); + var response = Substitute.For(); + page.GotoAsync(Arg.Any()).Returns(Task.FromResult(response)); + + var playwrightService = Substitute.For(); + playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + + var searchConfiguration = new SearchConfigurationBuilder { SubscriptionsStartingPageNumber = 1, SubscriptionsTotalPages = 1, }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var subscriptionsImagesListPage = new SubscriptionsImagesListPage(playwrightService, searchConfiguration); + + var contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + var failingPlaywrightService = Substitute.For(); + failingPlaywrightService.ConfigurePlaywrightAsync().ThrowsAsync(new PlaywrightException("navigation failed")); + var fileDetailRepository = Substitute.For(); + fileDetailRepository.ExistsAsync(Arg.Any()).Returns(false); + var imagePageService = new ImagePageService( + new ImagePage(failingPlaywrightService, scrapeConfiguration, new(), new()), + fileDetailRepository, + new FileClassificationService(contextFactory), + scrapeConfiguration, + System.TimeProvider.System, + new LoggerConfiguration().CreateLogger(), + Substitute.For(), + new(), + new NoOpDelayStrategy(), + Substitute.For(), + Substitute.For(), + new MockFileSystem(), + Substitute.For(), + Substitute.For()); + + var sut = new SubscriptionsWorkflow(subscriptionsImagesListPage, imagePageService, searchConfiguration, scrapeConfiguration.ScrapeDirectories, configurationSaver, new LoggerConfiguration().CreateLogger()); + + var exception = await Record.ExceptionAsync(() => sut.RunAsync(TestContext.Current.CancellationToken)); + + exception.ShouldBeOfType(); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenATopWallpapersWorkflowCompiles.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenATopWallpapersWorkflowCompiles.cs new file mode 100644 index 0000000..f69cf29 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenATopWallpapersWorkflowCompiles.cs @@ -0,0 +1,96 @@ +using NSubstitute.ExceptionExtensions; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Infrastructure.AppDb; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Pages; +using AStar.Dev.Wallpaper.Scrapper.Repositories; +using AStar.Dev.Wallpaper.Scrapper.Support; +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; +using AStar.Dev.Wallpaper.Scrapper.Workflows; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Playwright; +using Serilog; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Workflows; + +public sealed class GivenATopWallpapersWorkflowCompiles +{ + [Fact] + public async Task when_run_against_the_result_based_page_contract_then_no_exception_is_thrown() + { + var topWallpapersPage = Substitute.For(); + topWallpapersPage.LoadTopWallpapersPageAsync(Arg.Any()).Returns(Task.FromResult(Result.Success(global::AStar.Dev.FunctionalParadigm.Unit.Value))); + topWallpapersPage.PageInfoAsync().Returns(Task.FromResult(Result.Success(0))); + topWallpapersPage.GetImagePageLinksAsync().Returns(Task.FromResult(Result.Success, ScrapeError>([]))); + + var searchConfiguration = new SearchConfigurationBuilder { TopWallpapersStartingPageNumber = 1, TopWallpapersTotalPages = 0, }.Build(); + var contextFactory = Substitute.For>(); + var configurationSaver = new ConfigurationSaver(new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(), new LoggerConfiguration().CreateLogger(), contextFactory); + var imagePageService = new ImagePageService( + new ImagePage(Substitute.For(), new ScrapeConfigurationBuilder().Build(), new(), new()), + Substitute.For(), + new FileClassificationService(contextFactory), + new ScrapeConfigurationBuilder().Build(), + System.TimeProvider.System, + new LoggerConfiguration().CreateLogger(), + Substitute.For(), + new(), + new NoOpDelayStrategy(), + Substitute.For(), + Substitute.For(), + new MockFileSystem(), + Substitute.For(), + Substitute.For()); + + var sut = new TopWallpapersWorkflow(topWallpapersPage, imagePageService, searchConfiguration, configurationSaver, new LoggerConfiguration().CreateLogger()); + + var exception = await Record.ExceptionAsync(() => sut.RunAsync(TestContext.Current.CancellationToken)); + + exception.ShouldBeNull(); + } + + [Fact] + public async Task when_the_image_page_service_reports_a_failure_then_run_async_throws_an_invalid_operation_exception() + { + await using var connection = new SqliteConnection("Data Source=:memory:"); + await connection.OpenAsync(TestContext.Current.CancellationToken); + var options = new DbContextOptionsBuilder().UseSqlite(connection).Options; + await using (var seedContext = new AppDbContext(options)) await seedContext.Database.MigrateAsync(TestContext.Current.CancellationToken); + + var topWallpapersPage = Substitute.For(); + topWallpapersPage.LoadTopWallpapersPageAsync(Arg.Any()).Returns(Task.FromResult(Result.Success(global::AStar.Dev.FunctionalParadigm.Unit.Value))); + topWallpapersPage.PageInfoAsync().Returns(Task.FromResult(Result.Success(1))); + topWallpapersPage.GetImagePageLinksAsync().Returns(Task.FromResult(Result.Success, ScrapeError>(["https://example.test/w/12345",]))); + + var searchConfiguration = new SearchConfigurationBuilder { TopWallpapersStartingPageNumber = 1, TopWallpapersTotalPages = 1, }.Build(); + var contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); + var configurationSaver = new ConfigurationSaver(new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(), new LoggerConfiguration().CreateLogger(), contextFactory); + var failingPlaywrightService = Substitute.For(); + failingPlaywrightService.ConfigurePlaywrightAsync().ThrowsAsync(new PlaywrightException("navigation failed")); + var fileDetailRepository = Substitute.For(); + fileDetailRepository.ExistsAsync(Arg.Any()).Returns(false); + var imagePageService = new ImagePageService( + new ImagePage(failingPlaywrightService, new ScrapeConfigurationBuilder().Build(), new(), new()), + fileDetailRepository, + new FileClassificationService(contextFactory), + new ScrapeConfigurationBuilder().Build(), + System.TimeProvider.System, + new LoggerConfiguration().CreateLogger(), + Substitute.For(), + new(), + new NoOpDelayStrategy(), + Substitute.For(), + Substitute.For(), + new MockFileSystem(), + Substitute.For(), + Substitute.For()); + + var sut = new TopWallpapersWorkflow(topWallpapersPage, imagePageService, searchConfiguration, configurationSaver, new LoggerConfiguration().CreateLogger()); + + var exception = await Record.ExceptionAsync(() => sut.RunAsync(TestContext.Current.CancellationToken)); + + exception.ShouldBeOfType(); + } +}