diff --git a/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs b/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs index 2167ade..6aa7c43 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/App.axaml.cs @@ -49,6 +49,7 @@ public override void OnFrameworkInitializationCompleted() return ctx.ScrapeConfiguration.GetScrapeConfigurations().ToAppModel(); }) + .AddSingleton(sp => sp.GetRequiredService().SearchConfiguration) .AddSingleton() .AddSingleton() .AddSingleton(sp => @@ -91,6 +92,11 @@ public override void OnFrameworkInitializationCompleted() .AddSingleton() .AddTransient() .AddTransient() + .AddTransient() + .AddTransient() + .AddTransient() + .AddTransient() + .AddTransient() .AddTransient() .AddTransient() .AddTransient() diff --git a/src/AStar.Dev.Wallpaper.Scrapper/ScrapperConstants.cs b/src/AStar.Dev.Wallpaper.Scrapper/ScrapperConstants.cs index 89cd38b..c528a7c 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/ScrapperConstants.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/ScrapperConstants.cs @@ -8,8 +8,6 @@ public static class ScrapperConstants public const float ThumbnailCornerRadius = 20f; - public static readonly TimeSpan PageNavigationDelay = TimeSpan.FromSeconds(2); - public static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(10); public static readonly TimeSpan ImageAlreadyDownloadedDelay = TimeSpan.FromMilliseconds(500); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Support/RandomDelayStrategy.cs b/src/AStar.Dev.Wallpaper.Scrapper/Support/RandomDelayStrategy.cs index 5ffea6b..2f13abe 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Support/RandomDelayStrategy.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Support/RandomDelayStrategy.cs @@ -10,10 +10,13 @@ public Task DelayAsync(DelayKind delayKind, CancellationToken cancellationToken => delayKind switch { DelayKind.CategoryUpToDate => Task.Delay(TimeSpan.FromSeconds(new Random().Next(1, 5)), cancellationToken), - DelayKind.PageNavigation => Task.Delay(ScrapperConstants.PageNavigationDelay, cancellationToken), + DelayKind.PageNavigation => Task.Delay(TimeSpan.FromSeconds(RandomImagePauseSeconds()), cancellationToken), DelayKind.ImageAlreadyDownloaded => Task.Delay(ScrapperConstants.ImageAlreadyDownloadedDelay, cancellationToken), - DelayKind.BeforeImage => Task.Delay(TimeSpan.FromSeconds(Random.Shared.Next(scrapeConfiguration.SearchConfiguration.ImagePauseInSeconds, scrapeConfiguration.SearchConfiguration.ImagePauseInSeconds + 4)), cancellationToken), + DelayKind.BeforeImage => Task.Delay(TimeSpan.FromSeconds(RandomImagePauseSeconds()), cancellationToken), DelayKind.Retry => Task.Delay(ScrapperConstants.RetryDelay, cancellationToken), _ => throw new ArgumentOutOfRangeException(nameof(delayKind), delayKind, "Unrecognised delay kind."), }; + + private int RandomImagePauseSeconds() + => Random.Shared.Next(scrapeConfiguration.SearchConfiguration.ImagePauseInSeconds, scrapeConfiguration.SearchConfiguration.ImagePauseInSeconds + 4); } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/PagedScrapePlan.cs b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/PagedScrapePlan.cs new file mode 100644 index 0000000..2d8124b --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/PagedScrapePlan.cs @@ -0,0 +1,19 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Workflows; + +/// Describes one workflow's configuration of the shared page loop. +/// The first page number to visit. +/// The last page number to visit, inclusive. +/// Invoked with the current page number before the page is loaded, so the caller can record its own progress state. +/// Loads the page identified by the supplied page number. +/// Reads the image page links from the currently loaded page. +/// Processes the image page links read from the currently loaded page. +public record PagedScrapePlan( + int StartingPage, + int TotalPages, + Action RecordProgress, + Func>> LoadPageAsync, + Func, ScrapeError>>> GetLinksAsync, + Func, CancellationToken, Task>> ProcessLinksAsync); diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/PagedScrapePlanFactory.cs b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/PagedScrapePlanFactory.cs new file mode 100644 index 0000000..c2070a3 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/PagedScrapePlanFactory.cs @@ -0,0 +1,26 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Guard.Clauses; +using AStar.Dev.Wallpaper.Scrapper.Models; + +namespace AStar.Dev.Wallpaper.Scrapper.Workflows; + +/// Factory methods for creating instances of . +public static class PagedScrapePlanFactory +{ + /// Creates a . + public static PagedScrapePlan Create( + int startingPage, + int totalPages, + Action recordProgress, + Func>> loadPageAsync, + Func, ScrapeError>>> getLinksAsync, + Func, CancellationToken, Task>> processLinksAsync) + { + GuardAgainst.Null(recordProgress); + GuardAgainst.Null(loadPageAsync); + GuardAgainst.Null(getLinksAsync); + GuardAgainst.Null(processLinksAsync); + + return new(startingPage, totalPages, recordProgress, loadPageAsync, getLinksAsync, processLinksAsync); + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/PagedScrapeRunner.cs b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/PagedScrapeRunner.cs new file mode 100644 index 0000000..4ad1812 --- /dev/null +++ b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/PagedScrapeRunner.cs @@ -0,0 +1,35 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Guard.Clauses; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Support; + +namespace AStar.Dev.Wallpaper.Scrapper.Workflows; + +/// Runs the page loop shared by every paged scrape workflow: delay, record progress, save configuration, load the page, read its links, then process them. +public sealed class PagedScrapeRunner(ConfigurationSaver configurationSaver, IDelayStrategy delayStrategy) +{ + /// Runs from its starting page to its total page, inclusive, stopping at the first failing step. + public async Task> RunAsync(PagedScrapePlan plan, CancellationToken ct = default) + { + GuardAgainst.Null(plan); + + for (int pageNumber = plan.StartingPage; pageNumber <= plan.TotalPages; pageNumber++) + { + ct.ThrowIfCancellationRequested(); + await delayStrategy.DelayAsync(DelayKind.PageNavigation, ct).ConfigureAwait(false); + plan.RecordProgress(pageNumber); + + var pageResult = await configurationSaver.SaveUpdatedConfigurationAsync() + .BindAsync(_ => plan.LoadPageAsync(pageNumber)) + .BindAsync(_ => plan.GetLinksAsync()) + .BindAsync(links => plan.ProcessLinksAsync(links, ct)) + .ConfigureAwait(false); + + var pageFailed = pageResult.Match(_ => false, _ => true); + + if (pageFailed) return pageResult; + } + + return Unit.Value; + } +} diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchWorkflow.cs b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchWorkflow.cs index 94a5231..5ba3091 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchWorkflow.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SearchWorkflow.cs @@ -8,7 +8,7 @@ namespace AStar.Dev.Wallpaper.Scrapper.Workflows; -public sealed class SearchWorkflow(SearchResultsPage searchResultsPage, ScrapeConfiguration injectedScrapeConfiguration, ConfigurationSaver configurationSaver, ImagePageService imagePageService, IDirectoryHelper directoryHelper, ILogger logger, IDelayStrategy delayStrategy, TimeProvider timeProvider) +public sealed class SearchWorkflow(SearchResultsPage searchResultsPage, ScrapeConfiguration injectedScrapeConfiguration, ConfigurationSaver configurationSaver, ImagePageService imagePageService, IDirectoryHelper directoryHelper, ILogger logger, IDelayStrategy delayStrategy, TimeProvider timeProvider, PagedScrapeRunner pagedScrapeRunner) { private SearchProgress progress = null!; @@ -84,26 +84,22 @@ private async Task> ProcessAllCategoryPagesAsync(Categ long startTimestamp = timeProvider.GetTimestamp(); logger.Debug("About to visit the specific {Category} pages now...", searchCategory.Name); - for (int currentPageNumber = progress.SearchConfiguration.StartingPageNumber; currentPageNumber <= progress.SearchConfiguration.TotalPages; currentPageNumber++) - { - await delayStrategy.DelayAsync(DelayKind.PageNavigation, ct).ConfigureAwait(false); - logger.Debug("About to visit page {page} (of {totalPages}) for {Category} now...", currentPageNumber, progress.SearchConfiguration.TotalPages, searchCategory.Name); - progress = progress with { SearchConfiguration = progress.SearchConfiguration with { StartingPageNumber = currentPageNumber, }, }; - searchCategory.LastPageVisited = currentPageNumber; - - var pageResult = await configurationSaver.SaveUpdatedConfigurationAsync() - .BindAsync(_ => searchResultsPage.LoadSearchPageAsync(combinedSearchString, currentPageNumber)) - .BindAsync(_ => searchResultsPage.ImagePageLinksAsync()) - .BindAsync(links => imagePageService.GetTheImagePagesAsync(links, searchCategory.Id, searchCategory.Name, ct)) - .ConfigureAwait(false); - - var pageFailed = pageResult.Match(_ => false, _ => true); - - if (pageFailed) return pageResult; - } + var plan = PagedScrapePlanFactory.Create( + progress.SearchConfiguration.StartingPageNumber, + progress.SearchConfiguration.TotalPages, + pageNumber => RecordCategoryPageProgress(searchCategory, pageNumber), + pageNumber => searchResultsPage.LoadSearchPageAsync(combinedSearchString, pageNumber), + () => searchResultsPage.ImagePageLinksAsync(), + (links, innerCt) => imagePageService.GetTheImagePagesAsync(links, searchCategory.Id, searchCategory.Name, innerCt)); - logger.Information("Completed visiting the {Category}. Total time: {CategoryVisitDuration}", searchCategory.Name, timeProvider.GetElapsedTime(startTimestamp)); + return (await pagedScrapeRunner.RunAsync(plan, ct).ConfigureAwait(false)) + .Tap(_ => logger.Information("Completed visiting the {Category}. Total time: {CategoryVisitDuration}", searchCategory.Name, timeProvider.GetElapsedTime(startTimestamp))); + } - return Unit.Value; + private void RecordCategoryPageProgress(Category searchCategory, int pageNumber) + { + logger.Debug("About to visit page {page} (of {totalPages}) for {Category} now...", pageNumber, progress.SearchConfiguration.TotalPages, searchCategory.Name); + progress = progress with { SearchConfiguration = progress.SearchConfiguration with { StartingPageNumber = pageNumber, }, }; + searchCategory.LastPageVisited = pageNumber; } } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SubscriptionsWorkflow.cs b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SubscriptionsWorkflow.cs index 31c542d..87b1881 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SubscriptionsWorkflow.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/SubscriptionsWorkflow.cs @@ -3,79 +3,81 @@ using AStar.Dev.Wallpaper.Scrapper.Pages; using AStar.Dev.Wallpaper.Scrapper.Services; using AStar.Dev.Wallpaper.Scrapper.Support; -using Serilog.Core; +using Serilog; namespace AStar.Dev.Wallpaper.Scrapper.Workflows; public sealed class SubscriptionsWorkflow( SubscriptionsImagesListPage subscriptionsImagesListPage, ImagePageService imagePageService, - SearchConfiguration searchConfiguration, - ScrapeDirectories scrapeDirectories, + ScrapeConfiguration scrapeConfiguration, ConfigurationSaver configurationSaver, - Logger logger) + PagedScrapeRunner pagedScrapeRunner, + ILogger logger) { - private SearchConfiguration _searchConfiguration = searchConfiguration; - private ScrapeDirectories _scrapeDirectories = scrapeDirectories; + private const int FirstPageNumber = 1; - public async Task RunAsync(CancellationToken ct = default) + private SearchConfiguration searchConfiguration = scrapeConfiguration.SearchConfiguration; + private ScrapeDirectories scrapeDirectories = scrapeConfiguration.ScrapeDirectories; + + public Task> RunAsync(CancellationToken ct = default) + => RunSubscriptionsAsync(ct).LogFailure(logger); + + private async Task> RunSubscriptionsAsync(CancellationToken ct) { - try - { - await GetTheNewSubscriptionImagesAsync(ct).ConfigureAwait(false); - } - catch (Exception exception) when (exception is not OperationCanceledException) - { - logger.Error(exception.GetBaseException().Message); - throw; - } + searchConfiguration = searchConfiguration with { SubscriptionsStartingPageNumber = FirstPageNumber, }; + + await LoadStartingPageAsync().ConfigureAwait(false); + + return await subscriptionsImagesListPage.PageInfoAsync() + .BindAsync(pageInfo => ProcessSubscriptionsAsync(pageInfo, ct)) + .ConfigureAwait(false); } - private async Task GetTheNewSubscriptionImagesAsync(CancellationToken ct) + private async Task LoadStartingPageAsync() { - _searchConfiguration = _searchConfiguration with { SubscriptionsStartingPageNumber = 1 }; - var pageDetails = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(_searchConfiguration.SubscriptionsStartingPageNumber).ConfigureAwait(false); - var loadedSuccessfully = pageDetails.Match(_ => true, _ => false); + var loadResult = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(FirstPageNumber).ConfigureAwait(false); + var loadedSuccessfully = loadResult.Match(_ => true, _ => false); - if (!loadedSuccessfully) _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(1).ConfigureAwait(false); - - var pageInfo = (await subscriptionsImagesListPage.PageInfoAsync().ConfigureAwait(false)) - .Match(info => info, error => throw new InvalidOperationException(error.Message)); + if (!loadedSuccessfully) _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(FirstPageNumber).ConfigureAwait(false); + } - if (pageInfo.SubDirectoryName.Length > 0) _scrapeDirectories = _scrapeDirectories with { SubDirectoryName = pageInfo.SubDirectoryName }; + private async Task> ProcessSubscriptionsAsync(PageInfo pageInfo, CancellationToken ct) + { + if (pageInfo.SubDirectoryName.Length > 0) scrapeDirectories = scrapeDirectories with { SubDirectoryName = pageInfo.SubDirectoryName, }; - UpdateSearchTotalPagesIfRequired(pageInfo.PageCount); + if (searchConfiguration.SubscriptionsTotalPages != pageInfo.PageCount) searchConfiguration = searchConfiguration with { SubscriptionsTotalPages = pageInfo.PageCount, }; await configurationSaver.SaveUpdatedConfigurationAsync().ConfigureAwait(false); - for (int currentPageNumber = _searchConfiguration.SubscriptionsStartingPageNumber; - currentPageNumber <= _searchConfiguration.SubscriptionsTotalPages; - currentPageNumber++) - { - ct.ThrowIfCancellationRequested(); - int delay = Random.Shared.Next(_searchConfiguration.ImagePauseInSeconds, _searchConfiguration.ImagePauseInSeconds + 4); - await Task.Delay(TimeSpan.FromSeconds(delay), ct).ConfigureAwait(false); - _searchConfiguration = _searchConfiguration with { SubscriptionsStartingPageNumber = currentPageNumber }; - await configurationSaver.SaveUpdatedConfigurationAsync().ConfigureAwait(false); - logger.Information("Getting page {subscriptionPage} (of {totalPagesForSubscriptions}) now.", currentPageNumber, _searchConfiguration.SubscriptionsTotalPages); - _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(currentPageNumber).ConfigureAwait(false); - - var imagePageLinks = (await subscriptionsImagesListPage.GetImagePageLinksAsync().ConfigureAwait(false)) - .Match(links => links, error => throw new InvalidOperationException(error.Message)); - - _ = (await imagePageService.GetTheImagePagesAsync(imagePageLinks, "", pageInfo.SubDirectoryName, ct: ct).ConfigureAwait(false)) - .Match(unit => unit, error => throw new InvalidOperationException(error.Message)); - } - - if (pageInfo.PageCount > 0) - { - _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(1).ConfigureAwait(false); - _ = await subscriptionsImagesListPage.ClearAsync().ConfigureAwait(false); - } + var plan = PagedScrapePlanFactory.Create( + searchConfiguration.SubscriptionsStartingPageNumber, + searchConfiguration.SubscriptionsTotalPages, + _ => { }, + pageNumber => LoadSubscriptionsPageAsync(pageNumber, searchConfiguration.SubscriptionsTotalPages), + subscriptionsImagesListPage.GetImagePageLinksAsync, + (links, innerCt) => imagePageService.GetTheImagePagesAsync(links, string.Empty, pageInfo.SubDirectoryName, innerCt)); + + return await pagedScrapeRunner.RunAsync(plan, ct) + .BindAsync(_ => ClearSubscriptionsIfCompleteAsync(pageInfo)) + .ConfigureAwait(false); } - private void UpdateSearchTotalPagesIfRequired(int pageCount) + private async Task> LoadSubscriptionsPageAsync(int pageNumber, int totalPages) { - if (_searchConfiguration.SubscriptionsTotalPages != pageCount) _searchConfiguration = _searchConfiguration with { SubscriptionsTotalPages = pageCount }; + logger.Information("Getting page {subscriptionPage} (of {totalPagesForSubscriptions}) now.", pageNumber, totalPages); + _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(pageNumber).ConfigureAwait(false); + + return Unit.Value; + } + + private async Task> ClearSubscriptionsIfCompleteAsync(PageInfo pageInfo) + { + if (pageInfo.PageCount <= 0) return Unit.Value; + + _ = await subscriptionsImagesListPage.LoadSubscriptionResultsPageAsync(FirstPageNumber).ConfigureAwait(false); + _ = await subscriptionsImagesListPage.ClearAsync().ConfigureAwait(false); + + return Unit.Value; } } diff --git a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/TopWallpapersWorkflow.cs b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/TopWallpapersWorkflow.cs index 5498e15..41526eb 100644 --- a/src/AStar.Dev.Wallpaper.Scrapper/Workflows/TopWallpapersWorkflow.cs +++ b/src/AStar.Dev.Wallpaper.Scrapper/Workflows/TopWallpapersWorkflow.cs @@ -3,62 +3,66 @@ using AStar.Dev.Wallpaper.Scrapper.Pages; using AStar.Dev.Wallpaper.Scrapper.Services; using AStar.Dev.Wallpaper.Scrapper.Support; -using Serilog.Core; +using Serilog; namespace AStar.Dev.Wallpaper.Scrapper.Workflows; public sealed class TopWallpapersWorkflow( ITopWallpapersPage topWallpapersPage, ImagePageService imagePageService, - SearchConfiguration searchConfiguration, + ScrapeConfiguration scrapeConfiguration, ConfigurationSaver configurationSaver, - Logger logger) + PagedScrapeRunner pagedScrapeRunner, + ILogger logger) { - private SearchConfiguration _searchConfiguration = searchConfiguration; + private const int FirstPageNumber = 1; + private const string NoCategory = ""; - public async Task RunAsync(CancellationToken ct = default) + private SearchConfiguration searchConfiguration = scrapeConfiguration.SearchConfiguration; + + public Task> RunAsync(CancellationToken ct = default) + => RunTopWallpapersAsync(ct).LogFailure(logger); + + private async Task> RunTopWallpapersAsync(CancellationToken ct) { - try - { - await GetTheNewTopWallpapersAsync(ct).ConfigureAwait(false); - } - catch (Exception exception) when (exception is not OperationCanceledException) - { - logger.Error(exception.GetBaseException().Message); - throw; - } + await LoadStartingPageAsync().ConfigureAwait(false); + + return await topWallpapersPage.PageInfoAsync() + .BindAsync(pageCount => ProcessTopWallpapersAsync(pageCount, ct)) + .ConfigureAwait(false); } - private async Task GetTheNewTopWallpapersAsync(CancellationToken ct) + private async Task LoadStartingPageAsync() { - var pageDetails = await topWallpapersPage.LoadTopWallpapersPageAsync(_searchConfiguration.TopWallpapersStartingPageNumber).ConfigureAwait(false); - var loadedSuccessfully = pageDetails.Match(_ => true, _ => false); + var loadResult = await topWallpapersPage.LoadTopWallpapersPageAsync(searchConfiguration.TopWallpapersStartingPageNumber).ConfigureAwait(false); + var loadedSuccessfully = loadResult.Match(_ => true, _ => false); - if (!loadedSuccessfully) _ = await topWallpapersPage.LoadTopWallpapersPageAsync(1).ConfigureAwait(false); + if (!loadedSuccessfully) _ = await topWallpapersPage.LoadTopWallpapersPageAsync(FirstPageNumber).ConfigureAwait(false); + } - int pageCount = (await topWallpapersPage.PageInfoAsync().ConfigureAwait(false)) - .Match(count => count, error => throw new InvalidOperationException(error.Message)); + private async Task> ProcessTopWallpapersAsync(int pageCount, CancellationToken ct) + { logger.Information("There are a total of {TopWallpapersPageCount} pages for the Top Wallpapers.", pageCount); - _searchConfiguration = _searchConfiguration with { TopWallpapersTotalPages = pageCount }; + + if (searchConfiguration.TopWallpapersTotalPages != pageCount) searchConfiguration = searchConfiguration with { TopWallpapersTotalPages = pageCount, }; await configurationSaver.SaveUpdatedConfigurationAsync().ConfigureAwait(false); - for (int currentPageNumber = _searchConfiguration.TopWallpapersStartingPageNumber; - currentPageNumber <= _searchConfiguration.TopWallpapersTotalPages; - currentPageNumber++) - { - ct.ThrowIfCancellationRequested(); - int delay = Random.Shared.Next(_searchConfiguration.ImagePauseInSeconds, _searchConfiguration.ImagePauseInSeconds + 4); - await Task.Delay(TimeSpan.FromSeconds(delay), ct).ConfigureAwait(false); - _searchConfiguration = _searchConfiguration with { TopWallpapersStartingPageNumber = currentPageNumber }; - await configurationSaver.SaveUpdatedConfigurationAsync().ConfigureAwait(false); - _ = await topWallpapersPage.LoadTopWallpapersPageAsync(_searchConfiguration.TopWallpapersStartingPageNumber).ConfigureAwait(false); + var plan = PagedScrapePlanFactory.Create( + searchConfiguration.TopWallpapersStartingPageNumber, + searchConfiguration.TopWallpapersTotalPages, + _ => { }, + LoadTopWallpapersPageAsync, + topWallpapersPage.GetImagePageLinksAsync, + (links, innerCt) => imagePageService.GetTheImagePagesAsync(links, NoCategory, NoCategory, innerCt)); + + return await pagedScrapeRunner.RunAsync(plan, ct).ConfigureAwait(false); + } - var imagePageLinks = (await topWallpapersPage.GetImagePageLinksAsync().ConfigureAwait(false)) - .Match(links => links, error => throw new InvalidOperationException(error.Message)); + private async Task> LoadTopWallpapersPageAsync(int pageNumber) + { + _ = await topWallpapersPage.LoadTopWallpapersPageAsync(pageNumber).ConfigureAwait(false); - _ = (await imagePageService.GetTheImagePagesAsync(imagePageLinks, "", "", ct: ct).ConfigureAwait(false)) - .Match(unit => unit, error => throw new InvalidOperationException(error.Message)); - } + return Unit.Value; } } diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/ScrapperConstantsShould.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/ScrapperConstantsShould.cs index 7d5cf68..e011bbf 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/ScrapperConstantsShould.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/ScrapperConstantsShould.cs @@ -11,9 +11,6 @@ public sealed class ScrapperConstantsShould [Fact] public void HaveTheExpectedThumbnailCornerRadius() => ScrapperConstants.ThumbnailCornerRadius.ShouldBe(20f); - [Fact] - public void HaveTheExpectedPageNavigationDelay() => ScrapperConstants.PageNavigationDelay.ShouldBe(TimeSpan.FromSeconds(2)); - [Fact] public void HaveTheExpectedRetryDelay() => ScrapperConstants.RetryDelay.ShouldBe(TimeSpan.FromSeconds(10)); } diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheRandomDelayStrategy.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheRandomDelayStrategy.cs index 38a5c6b..525666e 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheRandomDelayStrategy.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Support/GivenTheRandomDelayStrategy.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using AStar.Dev.Wallpaper.Scrapper.Support; using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; @@ -20,4 +21,22 @@ public async Task when_the_cancellation_token_is_already_cancelled_then_delay_as await Should.ThrowAsync(() => sut.DelayAsync(delayKind, cts.Token)); } + + [Fact] + public async Task when_page_navigation_is_requested_with_a_zero_second_pause_configured_then_it_can_complete_in_under_two_seconds() + { + var zeroPauseStrategy = new RandomDelayStrategy(new ScrapeConfigurationBuilder { SearchConfiguration = new SearchConfigurationBuilder { ImagePauseInSeconds = 0, }.Build(), }.Build()); + var completedUnderTwoSeconds = false; + + for (int attempt = 0; attempt < 10 && !completedUnderTwoSeconds; attempt++) + { + var stopwatch = Stopwatch.StartNew(); + await zeroPauseStrategy.DelayAsync(DelayKind.PageNavigation, TestContext.Current.CancellationToken); + stopwatch.Stop(); + + completedUnderTwoSeconds = stopwatch.Elapsed < TimeSpan.FromSeconds(2); + } + + completedUnderTwoSeconds.ShouldBeTrue(); + } } diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenAPagedScrapeRunner.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenAPagedScrapeRunner.cs new file mode 100644 index 0000000..0111b22 --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenAPagedScrapeRunner.cs @@ -0,0 +1,304 @@ +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Infrastructure.AppDb; +using AStar.Dev.Infrastructure.AppDb.Entities; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Support; +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; +using AStar.Dev.Wallpaper.Scrapper.Workflows; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Serilog; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Workflows; + +public sealed class GivenAPagedScrapeRunner : IAsyncLifetime +{ + private SqliteConnection connection = null!; + private DbContextOptions options = null!; + + public async ValueTask InitializeAsync() + { + connection = new SqliteConnection("Data Source=:memory:"); + await connection.OpenAsync(); + + options = new DbContextOptionsBuilder().UseSqlite(connection).Options; + + await using var seedContext = new AppDbContext(options); + await seedContext.Database.MigrateAsync(); + + seedContext.ScrapeConfiguration.Add(new ScrapeConfigurationEntity + { + ConnectionStrings = new ConnectionStringsEntity { Sqlite = "Data Source=test.db", }, + UserConfiguration = new UserConfigurationEntity { LoginEmailAddress = "user@example.com", Username = "user", Password = "password", SessionCookie = "cookie", }, + SearchConfiguration = new SearchConfigurationEntity { BaseUrl = new Uri("https://example.com"), }, + ScrapeDirectories = new ScrapeDirectoriesEntity { RootDirectory = "root-directory", }, + }); + await seedContext.SaveChangesAsync(); + } + + public async ValueTask DisposeAsync() => await connection.DisposeAsync(); + + private sealed class RecordingDelayStrategy(List events) : IDelayStrategy + { + public Task DelayAsync(DelayKind delayKind, CancellationToken cancellationToken = default) + { + events.Add("delay"); + + return Task.CompletedTask; + } + } + + private PagedScrapeRunner BuildSut(IDelayStrategy delayStrategy, out IDbContextFactory contextFactory) + { + contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); + var configurationSaver = new ConfigurationSaver(new ScrapeConfigurationBuilder().Build(), new LoggerConfiguration().CreateLogger(), contextFactory); + + return new PagedScrapeRunner(configurationSaver, delayStrategy); + } + + private static Task> OkUnit() => Task.FromResult(Result.Success(global::AStar.Dev.FunctionalParadigm.Unit.Value)); + + private static Task, ScrapeError>> OkLinks() => Task.FromResult(Result.Success, ScrapeError>([])); + + [Fact] + public async Task when_run_then_pages_are_loaded_from_the_starting_page_to_the_total_page_inclusive_in_order() + { + List loadedPages = []; + var plan = PagedScrapePlanFactory.Create( + 1, + 3, + _ => { }, + page => + { + loadedPages.Add(page); + + return OkUnit(); + }, + OkLinks, + (_, _) => OkUnit()); + var sut = BuildSut(new NoOpDelayStrategy(), out _); + + await sut.RunAsync(plan, TestContext.Current.CancellationToken); + + loadedPages.ShouldBe([1, 2, 3,]); + } + + [Fact] + public async Task when_run_then_each_page_executes_delay_then_record_progress_then_save_then_load_then_get_links_then_process_links_in_order() + { + List events = []; + var contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => + { + events.Add("save"); + + return Task.FromResult(new AppDbContext(options)); + }); + var configurationSaver = new ConfigurationSaver(new ScrapeConfigurationBuilder().Build(), new LoggerConfiguration().CreateLogger(), contextFactory); + var sut = new PagedScrapeRunner(configurationSaver, new RecordingDelayStrategy(events)); + + var plan = PagedScrapePlanFactory.Create( + 1, + 2, + page => events.Add($"record:{page}"), + page => + { + events.Add($"load:{page}"); + + return OkUnit(); + }, + () => + { + events.Add($"links:{events.Count(e => e.StartsWith("load:", StringComparison.Ordinal))}"); + + return OkLinks(); + }, + (_, _) => + { + events.Add($"process:{events.Count(e => e.StartsWith("load:", StringComparison.Ordinal))}"); + + return OkUnit(); + }); + + await sut.RunAsync(plan, TestContext.Current.CancellationToken); + + events.ShouldBe(["delay", "record:1", "save", "load:1", "links:1", "process:1", "delay", "record:2", "save", "load:2", "links:2", "process:2",]); + } + + [Fact] + public async Task when_run_then_the_delay_strategy_is_consulted_once_per_page() + { + var plan = PagedScrapePlanFactory.Create(1, 3, _ => { }, _ => OkUnit(), OkLinks, (_, _) => OkUnit()); + var delayStrategy = Substitute.For(); + delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + var sut = BuildSut(delayStrategy, out _); + + await sut.RunAsync(plan, TestContext.Current.CancellationToken); + + await delayStrategy.Received(3).DelayAsync(DelayKind.PageNavigation, Arg.Any()); + } + + [Fact] + public async Task when_run_then_the_configuration_is_saved_once_per_page() + { + var plan = PagedScrapePlanFactory.Create(1, 3, _ => { }, _ => OkUnit(), OkLinks, (_, _) => OkUnit()); + var sut = BuildSut(new NoOpDelayStrategy(), out var contextFactory); + + await sut.RunAsync(plan, TestContext.Current.CancellationToken); + + await contextFactory.Received(3).CreateDbContextAsync(Arg.Any()); + } + + [Fact] + public async Task when_a_page_load_fails_then_later_pages_are_not_loaded() + { + List loadedPages = []; + var loadFailure = ScrapeErrorFactory.CreatePageLoadFailed("https://example.test/page/2", "navigation failed"); + var plan = PagedScrapePlanFactory.Create( + 1, + 3, + _ => { }, + page => + { + loadedPages.Add(page); + + return page == 2 ? Task.FromResult(Result.Failure(loadFailure)) : OkUnit(); + }, + OkLinks, + (_, _) => OkUnit()); + var sut = BuildSut(new NoOpDelayStrategy(), out _); + + var result = await sut.RunAsync(plan, TestContext.Current.CancellationToken); + + loadedPages.ShouldBe([1, 2,]); + result.ShouldBeOfType>().Error.ShouldBe(loadFailure); + } + + [Fact] + public async Task when_the_get_links_step_fails_then_process_links_is_not_called() + { + var linksFailure = ScrapeErrorFactory.CreatePageParseFailed(null, "could not parse links"); + var processLinksCalled = false; + var plan = PagedScrapePlanFactory.Create( + 1, + 2, + _ => { }, + _ => OkUnit(), + () => Task.FromResult(Result.Failure, ScrapeError>(linksFailure)), + (_, _) => + { + processLinksCalled = true; + + return OkUnit(); + }); + var sut = BuildSut(new NoOpDelayStrategy(), out _); + + var result = await sut.RunAsync(plan, TestContext.Current.CancellationToken); + + processLinksCalled.ShouldBeFalse(); + result.ShouldBeOfType>().Error.ShouldBe(linksFailure); + } + + [Fact] + public async Task when_the_process_links_step_fails_then_the_failure_is_returned_and_later_pages_are_not_loaded() + { + List loadedPages = []; + var processFailure = ScrapeErrorFactory.CreateImageDownloadFailed("https://example.test/image.jpg", "download failed"); + var plan = PagedScrapePlanFactory.Create( + 1, + 2, + _ => { }, + page => + { + loadedPages.Add(page); + + return OkUnit(); + }, + OkLinks, + (_, _) => Task.FromResult(Result.Failure(processFailure))); + var sut = BuildSut(new NoOpDelayStrategy(), out _); + + var result = await sut.RunAsync(plan, TestContext.Current.CancellationToken); + + loadedPages.ShouldBe([1,]); + result.ShouldBeOfType>().Error.ShouldBe(processFailure); + } + + [Fact] + public async Task when_the_total_page_is_before_the_starting_page_then_the_result_is_a_success_without_any_page_work() + { + var pageWorkHappened = false; + var plan = PagedScrapePlanFactory.Create( + 5, + 1, + _ => pageWorkHappened = true, + _ => + { + pageWorkHappened = true; + + return OkUnit(); + }, + () => + { + pageWorkHappened = true; + + return OkLinks(); + }, + (_, _) => + { + pageWorkHappened = true; + + return OkUnit(); + }); + var delayStrategy = Substitute.For(); + delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + var sut = BuildSut(delayStrategy, out var contextFactory); + + var result = await sut.RunAsync(plan, TestContext.Current.CancellationToken); + + result.ShouldBeOfType>(); + pageWorkHappened.ShouldBeFalse(); + await delayStrategy.DidNotReceive().DelayAsync(Arg.Any(), Arg.Any()); + await contextFactory.DidNotReceive().CreateDbContextAsync(Arg.Any()); + } + + [Fact] + public async Task when_the_cancellation_token_is_already_cancelled_then_run_async_throws_and_no_page_work_happens() + { + var pageWorkHappened = false; + var plan = PagedScrapePlanFactory.Create( + 1, + 1, + _ => pageWorkHappened = true, + _ => + { + pageWorkHappened = true; + + return OkUnit(); + }, + () => + { + pageWorkHappened = true; + + return OkLinks(); + }, + (_, _) => + { + pageWorkHappened = true; + + return OkUnit(); + }); + var delayStrategy = Substitute.For(); + delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + var sut = BuildSut(delayStrategy, out var contextFactory); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + + await Should.ThrowAsync(() => sut.RunAsync(plan, cts.Token)); + + pageWorkHappened.ShouldBeFalse(); + await delayStrategy.DidNotReceive().DelayAsync(Arg.Any(), Arg.Any()); + await contextFactory.DidNotReceive().CreateDbContextAsync(Arg.Any()); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowThatFails.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowThatFails.cs index c9786d4..683a43c 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowThatFails.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowThatFails.cs @@ -70,7 +70,7 @@ private SearchWorkflow BuildSut(IPage page, ILogger logger) var fileClassificationService = new FileClassificationService(contextFactory); var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), new NoOpDelayStrategy(), Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For(), Substitute.For()); - return new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), logger, new NoOpDelayStrategy(), System.TimeProvider.System); + return new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), logger, new NoOpDelayStrategy(), System.TimeProvider.System, new PagedScrapeRunner(configurationSaver, new NoOpDelayStrategy())); } [Fact] diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithADelayStrategy.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithADelayStrategy.cs index 0c29b03..abe08cc 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithADelayStrategy.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithADelayStrategy.cs @@ -93,7 +93,7 @@ public async Task when_a_category_is_up_to_date_then_the_delay_strategy_receives delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For(), Substitute.For()); - var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), delayStrategy, System.TimeProvider.System); + var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), delayStrategy, System.TimeProvider.System, new PagedScrapeRunner(configurationSaver, delayStrategy)); await sut.RunAsync(TestContext.Current.CancellationToken); @@ -127,7 +127,7 @@ public async Task when_a_category_is_not_up_to_date_then_the_delay_strategy_rece delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For(), Substitute.For()); - var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), delayStrategy, System.TimeProvider.System); + var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), delayStrategy, System.TimeProvider.System, new PagedScrapeRunner(configurationSaver, delayStrategy)); await sut.RunAsync(TestContext.Current.CancellationToken); @@ -160,7 +160,7 @@ public async Task when_the_delay_strategy_is_a_zero_delay_substitute_then_run_as var delayStrategy = new NoOpDelayStrategy(); var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), delayStrategy, Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For(), Substitute.For()); - var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), delayStrategy, System.TimeProvider.System); + var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), delayStrategy, System.TimeProvider.System, new PagedScrapeRunner(configurationSaver, delayStrategy)); var stopwatch = Stopwatch.StartNew(); await sut.RunAsync(TestContext.Current.CancellationToken); diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithAMidListSearchString.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithAMidListSearchString.cs index df30886..ae3bc27 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithAMidListSearchString.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithAMidListSearchString.cs @@ -56,7 +56,7 @@ public async Task when_run_then_categories_before_the_matching_category_are_not_ var fileClassificationService = new FileClassificationService(contextFactory); var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), new NoOpDelayStrategy(), Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For(), Substitute.For()); - var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), new NoOpDelayStrategy(), System.TimeProvider.System); + var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, Substitute.For(), Substitute.For(), new NoOpDelayStrategy(), System.TimeProvider.System, new PagedScrapeRunner(configurationSaver, new NoOpDelayStrategy())); await sut.RunAsync(TestContext.Current.CancellationToken); diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithANonEmptySubDirectory.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithANonEmptySubDirectory.cs index f89add7..35fae2b 100644 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithANonEmptySubDirectory.cs +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASearchWorkflowWithANonEmptySubDirectory.cs @@ -88,7 +88,7 @@ public async Task when_the_page_reports_a_non_empty_sub_directory_name_then_the_ var imagePageService = new ImagePageService(imagePage, Substitute.For(), fileClassificationService, scrapeConfiguration, System.TimeProvider.System, new LoggerConfiguration().CreateLogger(), Substitute.For(), new(), new NoOpDelayStrategy(), Substitute.For(), Substitute.For(), new MockFileSystem(), Substitute.For(), Substitute.For()); var directoryHelper = Substitute.For(); - var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, directoryHelper, Substitute.For(), new NoOpDelayStrategy(), System.TimeProvider.System); + var sut = new SearchWorkflow(searchResultsPage, scrapeConfiguration, configurationSaver, imagePageService, directoryHelper, Substitute.For(), new NoOpDelayStrategy(), System.TimeProvider.System, new PagedScrapeRunner(configurationSaver, new NoOpDelayStrategy())); await sut.RunAsync(TestContext.Current.CancellationToken); diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASubscriptionsWorkflow.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASubscriptionsWorkflow.cs new file mode 100644 index 0000000..db22ecc --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASubscriptionsWorkflow.cs @@ -0,0 +1,230 @@ +using NSubstitute.ExceptionExtensions; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Infrastructure.AppDb; +using AStar.Dev.Infrastructure.AppDb.Entities; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Pages; +using AStar.Dev.Wallpaper.Scrapper.Repositories; +using AStar.Dev.Wallpaper.Scrapper.Support; +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; +using AStar.Dev.Wallpaper.Scrapper.Workflows; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Playwright; +using Serilog; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Workflows; + +public sealed class GivenASubscriptionsWorkflow : IAsyncLifetime +{ + private SqliteConnection connection = null!; + private DbContextOptions options = null!; + + public async ValueTask InitializeAsync() + { + connection = new SqliteConnection("Data Source=:memory:"); + await connection.OpenAsync(); + + options = new DbContextOptionsBuilder().UseSqlite(connection).Options; + + await using var seedContext = new AppDbContext(options); + await seedContext.Database.MigrateAsync(); + + seedContext.ScrapeConfiguration.Add(new ScrapeConfigurationEntity + { + ConnectionStrings = new ConnectionStringsEntity { Sqlite = "Data Source=test.db", }, + UserConfiguration = new UserConfigurationEntity { LoginEmailAddress = "user@example.com", Username = "user", Password = "password", SessionCookie = "cookie", }, + SearchConfiguration = new SearchConfigurationEntity { BaseUrl = new Uri("https://example.com"), }, + ScrapeDirectories = new ScrapeDirectoriesEntity { RootDirectory = "root-directory", }, + }); + await seedContext.SaveChangesAsync(); + } + + public async ValueTask DisposeAsync() => await connection.DisposeAsync(); + + private IDbContextFactory BuildWorkingContextFactory() + { + var contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); + + return contextFactory; + } + + private static (IPage Page, IPlaywrightService PlaywrightService, ILocator ClearAllLocator) BuildPlaywright(string headerText, IReadOnlyList imageLinks) + { + var response = Substitute.For(); + var page = Substitute.For(); + page.GotoAsync(Arg.Any()).Returns(Task.FromResult(response)); + + var headerLocator = Substitute.For(); + headerLocator.TextContentAsync().Returns(Task.FromResult(headerText)); + page.GetByText(Arg.Any(), Arg.Any()).Returns(headerLocator); + + var linksLocator = Substitute.For(); + linksLocator.AllAsync().Returns(Task.FromResult(imageLinks)); + page.GetByRole(AriaRole.Link).Returns(linksLocator); + + var divLocator = Substitute.For(); + page.Locator("div").Returns(divLocator); + var filteredLocator = Substitute.For(); + divLocator.Filter(Arg.Any()).Returns(filteredLocator); + var clearAllLocator = Substitute.For(); + filteredLocator.GetByRole(AriaRole.Link, Arg.Any()).Returns(clearAllLocator); + clearAllLocator.ClickAsync().Returns(Task.CompletedTask); + + var playwrightService = Substitute.For(); + playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); + + return (page, playwrightService, clearAllLocator); + } + + private ImagePageService BuildImagePageService(IPlaywrightService playwrightService, ScrapeConfiguration scrapeConfiguration, IDirectoryHelper directoryHelper, IImageRetriever imageRetriever) + => new( + new ImagePage(playwrightService, scrapeConfiguration, new(), new()), + Substitute.For(), + new FileClassificationService(BuildWorkingContextFactory()), + scrapeConfiguration, + System.TimeProvider.System, + new LoggerConfiguration().CreateLogger(), + directoryHelper, + new(), + new NoOpDelayStrategy(), + imageRetriever, + Substitute.For(), + new MockFileSystem(), + Substitute.For(), + Substitute.For()); + + [Fact] + public async Task when_run_against_a_working_page_with_zero_total_pages_then_the_result_is_a_success() + { + var (_, playwrightService, _) = BuildPlaywright("0 New Subscription Wallpapers", []); + var searchConfiguration = new SearchConfigurationBuilder { SubscriptionsStartingPageNumber = 1, SubscriptionsTotalPages = 0, }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var subscriptionsImagesListPage = new SubscriptionsImagesListPage(playwrightService, searchConfiguration); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), Substitute.For>()); + var pagedScrapeRunner = new PagedScrapeRunner(configurationSaver, new NoOpDelayStrategy()); + var imagePageService = BuildImagePageService(playwrightService, scrapeConfiguration, Substitute.For(), Substitute.For()); + + var sut = new SubscriptionsWorkflow(subscriptionsImagesListPage, imagePageService, scrapeConfiguration, configurationSaver, pagedScrapeRunner, new LoggerConfiguration().CreateLogger()); + + var result = await sut.RunAsync(TestContext.Current.CancellationToken); + + result.ShouldBeOfType>(); + } + + [Fact] + public async Task when_the_image_page_service_reports_a_failure_then_run_async_returns_a_failure_result() + { + var linkLocator = Substitute.For(); + linkLocator.GetAttributeAsync("href").Returns(Task.FromResult("https://example.test/w/12345")); + var (_, playwrightService, _) = BuildPlaywright("1 New Subscription Wallpapers", [linkLocator,]); + + var searchConfiguration = new SearchConfigurationBuilder { SubscriptionsStartingPageNumber = 1, SubscriptionsTotalPages = 1, }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var subscriptionsImagesListPage = new SubscriptionsImagesListPage(playwrightService, searchConfiguration); + + var contextFactory = BuildWorkingContextFactory(); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + var pagedScrapeRunner = new PagedScrapeRunner(configurationSaver, new NoOpDelayStrategy()); + + var failingPlaywrightService = Substitute.For(); + failingPlaywrightService.ConfigurePlaywrightAsync().ThrowsAsync(new PlaywrightException("navigation failed")); + var imagePageService = BuildImagePageService(failingPlaywrightService, scrapeConfiguration, Substitute.For(), Substitute.For()); + + var sut = new SubscriptionsWorkflow(subscriptionsImagesListPage, imagePageService, scrapeConfiguration, configurationSaver, pagedScrapeRunner, new LoggerConfiguration().CreateLogger()); + + var result = await sut.RunAsync(TestContext.Current.CancellationToken); + + result.ShouldBeOfType>(); + } + + [Fact] + public async Task when_the_page_count_is_greater_than_zero_then_the_clear_all_locator_is_clicked_after_the_loop() + { + var (_, playwrightService, clearAllLocator) = BuildPlaywright("1 New Subscription Wallpapers", []); + var searchConfiguration = new SearchConfigurationBuilder { SubscriptionsStartingPageNumber = 1, SubscriptionsTotalPages = 1, }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var subscriptionsImagesListPage = new SubscriptionsImagesListPage(playwrightService, searchConfiguration); + + var contextFactory = BuildWorkingContextFactory(); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + var pagedScrapeRunner = new PagedScrapeRunner(configurationSaver, new NoOpDelayStrategy()); + var imagePageService = BuildImagePageService(playwrightService, scrapeConfiguration, Substitute.For(), Substitute.For()); + + var sut = new SubscriptionsWorkflow(subscriptionsImagesListPage, imagePageService, scrapeConfiguration, configurationSaver, pagedScrapeRunner, new LoggerConfiguration().CreateLogger()); + + await sut.RunAsync(TestContext.Current.CancellationToken); + + await clearAllLocator.Received(1).ClickAsync(); + } + + [Fact] + public async Task when_the_page_count_is_zero_then_the_clear_all_locator_is_not_clicked() + { + var (_, playwrightService, clearAllLocator) = BuildPlaywright("0 New Subscription Wallpapers", []); + var searchConfiguration = new SearchConfigurationBuilder { SubscriptionsStartingPageNumber = 1, SubscriptionsTotalPages = 0, }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var subscriptionsImagesListPage = new SubscriptionsImagesListPage(playwrightService, searchConfiguration); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), Substitute.For>()); + var pagedScrapeRunner = new PagedScrapeRunner(configurationSaver, new NoOpDelayStrategy()); + var imagePageService = BuildImagePageService(playwrightService, scrapeConfiguration, Substitute.For(), Substitute.For()); + + var sut = new SubscriptionsWorkflow(subscriptionsImagesListPage, imagePageService, scrapeConfiguration, configurationSaver, pagedScrapeRunner, new LoggerConfiguration().CreateLogger()); + + await sut.RunAsync(TestContext.Current.CancellationToken); + + await clearAllLocator.DidNotReceive().ClickAsync(); + } + + [Fact] + public async Task when_the_header_reports_a_sub_directory_name_then_the_processed_links_use_it_as_the_category_name() + { + const string ExpectedSubDirectoryFragment = "New-Subscription-Wallpapers"; + var linkLocator = Substitute.For(); + linkLocator.GetAttributeAsync("href").Returns(Task.FromResult("https://example.test/w/12345")); + var (_, playwrightService, _) = BuildPlaywright("1 New Subscription Wallpapers", [linkLocator,]); + + var searchConfiguration = new SearchConfigurationBuilder { SubscriptionsStartingPageNumber = 1, SubscriptionsTotalPages = 1, }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var subscriptionsImagesListPage = new SubscriptionsImagesListPage(playwrightService, searchConfiguration); + + var contextFactory = BuildWorkingContextFactory(); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + var pagedScrapeRunner = new PagedScrapeRunner(configurationSaver, new NoOpDelayStrategy()); + + var directoryHelper = Substitute.For(); + var imageRetriever = Substitute.For(); + imageRetriever.GetImageAsync(Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(Result.Failure(ScrapeErrorFactory.CreateImageDownloadFailed("url", "boom")))); + var imagePageService = BuildImagePageService(playwrightService, scrapeConfiguration, directoryHelper, imageRetriever); + + var sut = new SubscriptionsWorkflow(subscriptionsImagesListPage, imagePageService, scrapeConfiguration, configurationSaver, pagedScrapeRunner, new LoggerConfiguration().CreateLogger()); + + await sut.RunAsync(TestContext.Current.CancellationToken); + + directoryHelper.Received(1).CreateDirectoryIfRequired(Arg.Is>(segments => segments.Any(segment => segment.Contains(ExpectedSubDirectoryFragment)))); + } + + [Fact] + public async Task when_run_then_the_delay_strategy_is_consulted_once_per_page() + { + var (_, playwrightService, _) = BuildPlaywright("25 New Subscription Wallpapers", []); + var searchConfiguration = new SearchConfigurationBuilder { SubscriptionsStartingPageNumber = 1, SubscriptionsTotalPages = 2, }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var subscriptionsImagesListPage = new SubscriptionsImagesListPage(playwrightService, searchConfiguration); + + var contextFactory = BuildWorkingContextFactory(); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + var delayStrategy = Substitute.For(); + delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + var pagedScrapeRunner = new PagedScrapeRunner(configurationSaver, delayStrategy); + var imagePageService = BuildImagePageService(playwrightService, scrapeConfiguration, Substitute.For(), Substitute.For()); + + var sut = new SubscriptionsWorkflow(subscriptionsImagesListPage, imagePageService, scrapeConfiguration, configurationSaver, pagedScrapeRunner, new LoggerConfiguration().CreateLogger()); + + await sut.RunAsync(TestContext.Current.CancellationToken); + + await delayStrategy.Received(2).DelayAsync(DelayKind.PageNavigation, Arg.Any()); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASubscriptionsWorkflowCompiles.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASubscriptionsWorkflowCompiles.cs deleted file mode 100644 index abe0057..0000000 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenASubscriptionsWorkflowCompiles.cs +++ /dev/null @@ -1,119 +0,0 @@ -using NSubstitute.ExceptionExtensions; -using AStar.Dev.Infrastructure.AppDb; -using AStar.Dev.Wallpaper.Scrapper.Models; -using AStar.Dev.Wallpaper.Scrapper.Pages; -using AStar.Dev.Wallpaper.Scrapper.Repositories; -using AStar.Dev.Wallpaper.Scrapper.Support; -using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; -using AStar.Dev.Wallpaper.Scrapper.Workflows; -using Microsoft.Data.Sqlite; -using Microsoft.EntityFrameworkCore; -using Microsoft.Playwright; -using Serilog; - -namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Workflows; - -public sealed class GivenASubscriptionsWorkflowCompiles -{ - [Fact] - public async Task when_run_against_the_result_based_page_contract_then_no_exception_is_thrown() - { - var headerLocator = Substitute.For(); - headerLocator.TextContentAsync().Returns(Task.FromResult("0 New Subscription Wallpapers")); - var linksLocator = Substitute.For(); - linksLocator.AllAsync().Returns(Task.FromResult>([])); - var page = Substitute.For(); - page.GetByText(Arg.Any(), Arg.Any()).Returns(headerLocator); - page.GetByRole(AriaRole.Link).Returns(linksLocator); - var response = Substitute.For(); - page.GotoAsync(Arg.Any()).Returns(Task.FromResult(response)); - - var playwrightService = Substitute.For(); - playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); - - var searchConfiguration = new SearchConfigurationBuilder { SubscriptionsStartingPageNumber = 1, SubscriptionsTotalPages = 0, }.Build(); - var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); - var subscriptionsImagesListPage = new SubscriptionsImagesListPage(playwrightService, searchConfiguration); - - var contextFactory = Substitute.For>(); - var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); - var imagePageService = new ImagePageService( - new ImagePage(playwrightService, scrapeConfiguration, new(), new()), - Substitute.For(), - new FileClassificationService(contextFactory), - scrapeConfiguration, - System.TimeProvider.System, - new LoggerConfiguration().CreateLogger(), - Substitute.For(), - new(), - new NoOpDelayStrategy(), - Substitute.For(), - Substitute.For(), - new MockFileSystem(), - Substitute.For(), - Substitute.For()); - - var sut = new SubscriptionsWorkflow(subscriptionsImagesListPage, imagePageService, searchConfiguration, scrapeConfiguration.ScrapeDirectories, configurationSaver, new LoggerConfiguration().CreateLogger()); - - var exception = await Record.ExceptionAsync(() => sut.RunAsync(TestContext.Current.CancellationToken)); - - exception.ShouldBeNull(); - } - - [Fact] - public async Task when_the_image_page_service_reports_a_failure_then_run_async_throws_an_invalid_operation_exception() - { - await using var connection = new SqliteConnection("Data Source=:memory:"); - await connection.OpenAsync(TestContext.Current.CancellationToken); - var options = new DbContextOptionsBuilder().UseSqlite(connection).Options; - await using (var seedContext = new AppDbContext(options)) await seedContext.Database.MigrateAsync(TestContext.Current.CancellationToken); - - var headerLocator = Substitute.For(); - headerLocator.TextContentAsync().Returns(Task.FromResult("1 New Subscription Wallpapers")); - var linkLocator = Substitute.For(); - linkLocator.GetAttributeAsync("href").Returns(Task.FromResult("https://example.test/w/12345")); - var linksLocator = Substitute.For(); - linksLocator.AllAsync().Returns(Task.FromResult>([linkLocator,])); - var page = Substitute.For(); - page.GetByText(Arg.Any(), Arg.Any()).Returns(headerLocator); - page.GetByRole(AriaRole.Link).Returns(linksLocator); - var response = Substitute.For(); - page.GotoAsync(Arg.Any()).Returns(Task.FromResult(response)); - - var playwrightService = Substitute.For(); - playwrightService.ConfigurePlaywrightAsync().Returns(Task.FromResult(page)); - - var searchConfiguration = new SearchConfigurationBuilder { SubscriptionsStartingPageNumber = 1, SubscriptionsTotalPages = 1, }.Build(); - var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); - var subscriptionsImagesListPage = new SubscriptionsImagesListPage(playwrightService, searchConfiguration); - - var contextFactory = Substitute.For>(); - contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); - var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); - var failingPlaywrightService = Substitute.For(); - failingPlaywrightService.ConfigurePlaywrightAsync().ThrowsAsync(new PlaywrightException("navigation failed")); - var fileDetailRepository = Substitute.For(); - fileDetailRepository.ExistsAsync(Arg.Any()).Returns(false); - var imagePageService = new ImagePageService( - new ImagePage(failingPlaywrightService, scrapeConfiguration, new(), new()), - fileDetailRepository, - new FileClassificationService(contextFactory), - scrapeConfiguration, - System.TimeProvider.System, - new LoggerConfiguration().CreateLogger(), - Substitute.For(), - new(), - new NoOpDelayStrategy(), - Substitute.For(), - Substitute.For(), - new MockFileSystem(), - Substitute.For(), - Substitute.For()); - - var sut = new SubscriptionsWorkflow(subscriptionsImagesListPage, imagePageService, searchConfiguration, scrapeConfiguration.ScrapeDirectories, configurationSaver, new LoggerConfiguration().CreateLogger()); - - var exception = await Record.ExceptionAsync(() => sut.RunAsync(TestContext.Current.CancellationToken)); - - exception.ShouldBeOfType(); - } -} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenATopWallpapersWorkflow.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenATopWallpapersWorkflow.cs new file mode 100644 index 0000000..efbcb3d --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenATopWallpapersWorkflow.cs @@ -0,0 +1,163 @@ +using NSubstitute.ExceptionExtensions; +using AStar.Dev.FunctionalParadigm; +using AStar.Dev.Infrastructure.AppDb; +using AStar.Dev.Infrastructure.AppDb.Entities; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Pages; +using AStar.Dev.Wallpaper.Scrapper.Repositories; +using AStar.Dev.Wallpaper.Scrapper.Support; +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; +using AStar.Dev.Wallpaper.Scrapper.Workflows; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.Playwright; +using Serilog; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Workflows; + +public sealed class GivenATopWallpapersWorkflow : IAsyncLifetime +{ + private SqliteConnection connection = null!; + private DbContextOptions options = null!; + + public async ValueTask InitializeAsync() + { + connection = new SqliteConnection("Data Source=:memory:"); + await connection.OpenAsync(); + + options = new DbContextOptionsBuilder().UseSqlite(connection).Options; + + await using var seedContext = new AppDbContext(options); + await seedContext.Database.MigrateAsync(); + + seedContext.ScrapeConfiguration.Add(new ScrapeConfigurationEntity + { + ConnectionStrings = new ConnectionStringsEntity { Sqlite = "Data Source=test.db", }, + UserConfiguration = new UserConfigurationEntity { LoginEmailAddress = "user@example.com", Username = "user", Password = "password", SessionCookie = "cookie", }, + SearchConfiguration = new SearchConfigurationEntity { BaseUrl = new Uri("https://example.com"), }, + ScrapeDirectories = new ScrapeDirectoriesEntity { RootDirectory = "root-directory", }, + }); + await seedContext.SaveChangesAsync(); + } + + public async ValueTask DisposeAsync() => await connection.DisposeAsync(); + + private IDbContextFactory BuildWorkingContextFactory() + { + var contextFactory = Substitute.For>(); + contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); + + return contextFactory; + } + + private ImagePageService BuildImagePageService(IPlaywrightService playwrightService, ScrapeConfiguration scrapeConfiguration, IFileDetailRepository fileDetailRepository) + => new( + new ImagePage(playwrightService, scrapeConfiguration, new(), new()), + fileDetailRepository, + new FileClassificationService(BuildWorkingContextFactory()), + scrapeConfiguration, + System.TimeProvider.System, + new LoggerConfiguration().CreateLogger(), + Substitute.For(), + new(), + new NoOpDelayStrategy(), + Substitute.For(), + Substitute.For(), + new MockFileSystem(), + Substitute.For(), + Substitute.For()); + + [Fact] + public async Task when_run_against_a_working_page_with_zero_total_pages_then_the_result_is_a_success() + { + var topWallpapersPage = Substitute.For(); + topWallpapersPage.LoadTopWallpapersPageAsync(Arg.Any()).Returns(Task.FromResult(Result.Success(global::AStar.Dev.FunctionalParadigm.Unit.Value))); + topWallpapersPage.PageInfoAsync().Returns(Task.FromResult(Result.Success(0))); + topWallpapersPage.GetImagePageLinksAsync().Returns(Task.FromResult(Result.Success, ScrapeError>([]))); + + var searchConfiguration = new SearchConfigurationBuilder { TopWallpapersStartingPageNumber = 1, TopWallpapersTotalPages = 0, }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), Substitute.For>()); + var pagedScrapeRunner = new PagedScrapeRunner(configurationSaver, new NoOpDelayStrategy()); + var imagePageService = BuildImagePageService(Substitute.For(), scrapeConfiguration, Substitute.For()); + + var sut = new TopWallpapersWorkflow(topWallpapersPage, imagePageService, scrapeConfiguration, configurationSaver, pagedScrapeRunner, new LoggerConfiguration().CreateLogger()); + + var result = await sut.RunAsync(TestContext.Current.CancellationToken); + + result.ShouldBeOfType>(); + } + + [Fact] + public async Task when_the_image_page_service_reports_a_failure_then_run_async_returns_a_failure_result() + { + var topWallpapersPage = Substitute.For(); + topWallpapersPage.LoadTopWallpapersPageAsync(Arg.Any()).Returns(Task.FromResult(Result.Success(global::AStar.Dev.FunctionalParadigm.Unit.Value))); + topWallpapersPage.PageInfoAsync().Returns(Task.FromResult(Result.Success(1))); + topWallpapersPage.GetImagePageLinksAsync().Returns(Task.FromResult(Result.Success, ScrapeError>(["https://example.test/w/12345",]))); + + var searchConfiguration = new SearchConfigurationBuilder { TopWallpapersStartingPageNumber = 1, TopWallpapersTotalPages = 1, }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var contextFactory = BuildWorkingContextFactory(); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + var pagedScrapeRunner = new PagedScrapeRunner(configurationSaver, new NoOpDelayStrategy()); + + var failingPlaywrightService = Substitute.For(); + failingPlaywrightService.ConfigurePlaywrightAsync().ThrowsAsync(new PlaywrightException("navigation failed")); + var fileDetailRepository = Substitute.For(); + fileDetailRepository.ExistsAsync(Arg.Any()).Returns(false); + var imagePageService = BuildImagePageService(failingPlaywrightService, scrapeConfiguration, fileDetailRepository); + + var sut = new TopWallpapersWorkflow(topWallpapersPage, imagePageService, scrapeConfiguration, configurationSaver, pagedScrapeRunner, new LoggerConfiguration().CreateLogger()); + + var result = await sut.RunAsync(TestContext.Current.CancellationToken); + + result.ShouldBeOfType>(); + } + + [Fact] + public async Task when_run_then_the_delay_strategy_is_consulted_once_per_page_reported_by_the_header() + { + var topWallpapersPage = Substitute.For(); + topWallpapersPage.LoadTopWallpapersPageAsync(Arg.Any()).Returns(Task.FromResult(Result.Success(global::AStar.Dev.FunctionalParadigm.Unit.Value))); + topWallpapersPage.PageInfoAsync().Returns(Task.FromResult(Result.Success(3))); + topWallpapersPage.GetImagePageLinksAsync().Returns(Task.FromResult(Result.Success, ScrapeError>([]))); + + var searchConfiguration = new SearchConfigurationBuilder { TopWallpapersStartingPageNumber = 1, TopWallpapersTotalPages = 1, }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var contextFactory = BuildWorkingContextFactory(); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), contextFactory); + var delayStrategy = Substitute.For(); + delayStrategy.DelayAsync(Arg.Any(), Arg.Any()).Returns(Task.CompletedTask); + var pagedScrapeRunner = new PagedScrapeRunner(configurationSaver, delayStrategy); + var imagePageService = BuildImagePageService(Substitute.For(), scrapeConfiguration, Substitute.For()); + + var sut = new TopWallpapersWorkflow(topWallpapersPage, imagePageService, scrapeConfiguration, configurationSaver, pagedScrapeRunner, new LoggerConfiguration().CreateLogger()); + + await sut.RunAsync(TestContext.Current.CancellationToken); + + await delayStrategy.Received(3).DelayAsync(DelayKind.PageNavigation, Arg.Any()); + } + + [Fact] + public async Task when_the_starting_page_fails_to_load_then_the_workflow_reloads_page_one() + { + var topWallpapersPage = Substitute.For(); + topWallpapersPage.LoadTopWallpapersPageAsync(5).Returns(Task.FromResult(Result.Failure(ScrapeErrorFactory.CreatePageLoadFailed("url", "boom")))); + topWallpapersPage.LoadTopWallpapersPageAsync(1).Returns(Task.FromResult(Result.Success(global::AStar.Dev.FunctionalParadigm.Unit.Value))); + topWallpapersPage.PageInfoAsync().Returns(Task.FromResult(Result.Success(0))); + topWallpapersPage.GetImagePageLinksAsync().Returns(Task.FromResult(Result.Success, ScrapeError>([]))); + + var searchConfiguration = new SearchConfigurationBuilder { TopWallpapersStartingPageNumber = 5, TopWallpapersTotalPages = 0, }.Build(); + var scrapeConfiguration = new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(); + var configurationSaver = new ConfigurationSaver(scrapeConfiguration, new LoggerConfiguration().CreateLogger(), Substitute.For>()); + var pagedScrapeRunner = new PagedScrapeRunner(configurationSaver, new NoOpDelayStrategy()); + var imagePageService = BuildImagePageService(Substitute.For(), scrapeConfiguration, Substitute.For()); + + var sut = new TopWallpapersWorkflow(topWallpapersPage, imagePageService, scrapeConfiguration, configurationSaver, pagedScrapeRunner, new LoggerConfiguration().CreateLogger()); + + await sut.RunAsync(TestContext.Current.CancellationToken); + + await topWallpapersPage.Received(1).LoadTopWallpapersPageAsync(1); + } +} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenATopWallpapersWorkflowCompiles.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenATopWallpapersWorkflowCompiles.cs deleted file mode 100644 index f69cf29..0000000 --- a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenATopWallpapersWorkflowCompiles.cs +++ /dev/null @@ -1,96 +0,0 @@ -using NSubstitute.ExceptionExtensions; -using AStar.Dev.FunctionalParadigm; -using AStar.Dev.Infrastructure.AppDb; -using AStar.Dev.Wallpaper.Scrapper.Models; -using AStar.Dev.Wallpaper.Scrapper.Pages; -using AStar.Dev.Wallpaper.Scrapper.Repositories; -using AStar.Dev.Wallpaper.Scrapper.Support; -using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; -using AStar.Dev.Wallpaper.Scrapper.Workflows; -using Microsoft.Data.Sqlite; -using Microsoft.EntityFrameworkCore; -using Microsoft.Playwright; -using Serilog; - -namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Workflows; - -public sealed class GivenATopWallpapersWorkflowCompiles -{ - [Fact] - public async Task when_run_against_the_result_based_page_contract_then_no_exception_is_thrown() - { - var topWallpapersPage = Substitute.For(); - topWallpapersPage.LoadTopWallpapersPageAsync(Arg.Any()).Returns(Task.FromResult(Result.Success(global::AStar.Dev.FunctionalParadigm.Unit.Value))); - topWallpapersPage.PageInfoAsync().Returns(Task.FromResult(Result.Success(0))); - topWallpapersPage.GetImagePageLinksAsync().Returns(Task.FromResult(Result.Success, ScrapeError>([]))); - - var searchConfiguration = new SearchConfigurationBuilder { TopWallpapersStartingPageNumber = 1, TopWallpapersTotalPages = 0, }.Build(); - var contextFactory = Substitute.For>(); - var configurationSaver = new ConfigurationSaver(new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(), new LoggerConfiguration().CreateLogger(), contextFactory); - var imagePageService = new ImagePageService( - new ImagePage(Substitute.For(), new ScrapeConfigurationBuilder().Build(), new(), new()), - Substitute.For(), - new FileClassificationService(contextFactory), - new ScrapeConfigurationBuilder().Build(), - System.TimeProvider.System, - new LoggerConfiguration().CreateLogger(), - Substitute.For(), - new(), - new NoOpDelayStrategy(), - Substitute.For(), - Substitute.For(), - new MockFileSystem(), - Substitute.For(), - Substitute.For()); - - var sut = new TopWallpapersWorkflow(topWallpapersPage, imagePageService, searchConfiguration, configurationSaver, new LoggerConfiguration().CreateLogger()); - - var exception = await Record.ExceptionAsync(() => sut.RunAsync(TestContext.Current.CancellationToken)); - - exception.ShouldBeNull(); - } - - [Fact] - public async Task when_the_image_page_service_reports_a_failure_then_run_async_throws_an_invalid_operation_exception() - { - await using var connection = new SqliteConnection("Data Source=:memory:"); - await connection.OpenAsync(TestContext.Current.CancellationToken); - var options = new DbContextOptionsBuilder().UseSqlite(connection).Options; - await using (var seedContext = new AppDbContext(options)) await seedContext.Database.MigrateAsync(TestContext.Current.CancellationToken); - - var topWallpapersPage = Substitute.For(); - topWallpapersPage.LoadTopWallpapersPageAsync(Arg.Any()).Returns(Task.FromResult(Result.Success(global::AStar.Dev.FunctionalParadigm.Unit.Value))); - topWallpapersPage.PageInfoAsync().Returns(Task.FromResult(Result.Success(1))); - topWallpapersPage.GetImagePageLinksAsync().Returns(Task.FromResult(Result.Success, ScrapeError>(["https://example.test/w/12345",]))); - - var searchConfiguration = new SearchConfigurationBuilder { TopWallpapersStartingPageNumber = 1, TopWallpapersTotalPages = 1, }.Build(); - var contextFactory = Substitute.For>(); - contextFactory.CreateDbContextAsync(Arg.Any()).Returns(_ => Task.FromResult(new AppDbContext(options))); - var configurationSaver = new ConfigurationSaver(new ScrapeConfigurationBuilder { SearchConfiguration = searchConfiguration, }.Build(), new LoggerConfiguration().CreateLogger(), contextFactory); - var failingPlaywrightService = Substitute.For(); - failingPlaywrightService.ConfigurePlaywrightAsync().ThrowsAsync(new PlaywrightException("navigation failed")); - var fileDetailRepository = Substitute.For(); - fileDetailRepository.ExistsAsync(Arg.Any()).Returns(false); - var imagePageService = new ImagePageService( - new ImagePage(failingPlaywrightService, new ScrapeConfigurationBuilder().Build(), new(), new()), - fileDetailRepository, - new FileClassificationService(contextFactory), - new ScrapeConfigurationBuilder().Build(), - System.TimeProvider.System, - new LoggerConfiguration().CreateLogger(), - Substitute.For(), - new(), - new NoOpDelayStrategy(), - Substitute.For(), - Substitute.For(), - new MockFileSystem(), - Substitute.For(), - Substitute.For()); - - var sut = new TopWallpapersWorkflow(topWallpapersPage, imagePageService, searchConfiguration, configurationSaver, new LoggerConfiguration().CreateLogger()); - - var exception = await Record.ExceptionAsync(() => sut.RunAsync(TestContext.Current.CancellationToken)); - - exception.ShouldBeOfType(); - } -} diff --git a/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenTheWorkflowServiceRegistrations.cs b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenTheWorkflowServiceRegistrations.cs new file mode 100644 index 0000000..ffd116b --- /dev/null +++ b/test/AStar.Dev.Wallpaper.Scrapper.Tests.Unit/Workflows/GivenTheWorkflowServiceRegistrations.cs @@ -0,0 +1,73 @@ +using System.IO.Abstractions; +using AStar.Dev.Infrastructure.AppDb; +using AStar.Dev.Wallpaper.Scrapper.DTOs; +using AStar.Dev.Wallpaper.Scrapper.Models; +using AStar.Dev.Wallpaper.Scrapper.Pages; +using AStar.Dev.Wallpaper.Scrapper.Repositories; +using AStar.Dev.Wallpaper.Scrapper.Support; +using AStar.Dev.Wallpaper.Scrapper.Tests.Unit.TestData; +using AStar.Dev.Wallpaper.Scrapper.Workflows; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Serilog; + +namespace AStar.Dev.Wallpaper.Scrapper.Tests.Unit.Workflows; + +public sealed class GivenTheWorkflowServiceRegistrations +{ + private static ServiceProvider BuildServiceProvider() + { + var logger = new LoggerConfiguration().CreateLogger(); + + var services = new ServiceCollection() + .AddSingleton(new ScrapeConfigurationBuilder().Build()) + .AddSingleton(sp => sp.GetRequiredService().SearchConfiguration) + .AddSingleton(logger) + .AddSingleton(logger) + .AddSingleton(new TagsToIgnoreCompletely()) + .AddSingleton(new TagsTextToIgnore()) + .AddSingleton(_ => Substitute.For>()) + .AddSingleton(_ => Substitute.For()) + .AddSingleton(_ => Substitute.For()) + .AddSingleton(_ => Substitute.For()) + .AddSingleton(_ => Substitute.For()) + .AddSingleton(_ => Substitute.For()) + .AddSingleton(_ => Substitute.For()) + .AddSingleton(_ => Substitute.For()) + .AddSingleton(new MockFileSystem()) + .AddSingleton() + .AddSingleton() + .AddTransient(_ => System.TimeProvider.System) + .AddTransient() + .AddTransient() + .AddTransient() + .AddTransient() + .AddTransient() + .AddTransient() + .AddTransient() + .AddTransient() + .AddTransient(); + + return services.BuildServiceProvider(); + } + + [Fact] + public void when_resolving_the_subscriptions_workflow_then_resolution_succeeds() + { + using var serviceProvider = BuildServiceProvider(); + + var subscriptionsWorkflow = serviceProvider.GetRequiredService(); + + subscriptionsWorkflow.ShouldNotBeNull(); + } + + [Fact] + public void when_resolving_the_top_wallpapers_workflow_then_resolution_succeeds() + { + using var serviceProvider = BuildServiceProvider(); + + var topWallpapersWorkflow = serviceProvider.GetRequiredService(); + + topWallpapersWorkflow.ShouldNotBeNull(); + } +}