refactor(wallpaper-scraper): Complete code review implementation - functional paradigm and DB optimization

.claude/settings.json

@@ -43,7 +43,16 @@
         "hooks": [
           {
             "type": "command",
-            "command": "CMD=$(python3 -c \"import json,sys; d=json.load(sys.stdin); print(d.get('tool_input',d).get('command',''))\" 2>/dev/null || true); case \"$CMD\" in *grep*|*rg\\ *|*ripgrep*|*find\\ *|*fd\\ *|*ack\\ *|*ag\\ *)   [ -f graphify-out/graph.json ] &&   echo '{\"hookSpecificOutput\":{\"hookEventName\":\"PreToolUse\",\"additionalContext\":\"graphify: Knowledge graph exists. Read graphify-out/GRAPH_REPORT.md for god nodes and community structure before searching raw files.\"}}'   || true ;; esac"
+            "command": "/home/jbarden/.local/bin/graphify hook-guard search"
+          }
+        ]
+      },
+      {
+        "matcher": "Read|Glob",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "/home/jbarden/.local/bin/graphify hook-guard read"
           }
         ]
       }
@@ -55,6 +64,7 @@
     "padding": 2
   },
   "enabledPlugins": {
-    "csharp-lsp@claude-plugins-official": true
+    "csharp-lsp@claude-plugins-official": true,
+    "serena@claude-plugins-official": true
   }
-}
+}

CLAUDE.md

@@ -102,5 +102,10 @@ Always use `gh` CLI for all GitHub operations. Never use MCP GitHub - not config
 
 ## graphify
 
-Knowledge graph at `graphify-out/`. Before architecture questions, read `graphify-out/GRAPH_REPORT.md`.
-Use `graphify query/path/explain` for cross-module questions. After modifying code, run `graphify update .`.
+This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships.
+
+Rules:
+- For codebase questions, first run `graphify query "<question>"` when graphify-out/graph.json exists. Use `graphify path "<A>" "<B>"` for relationships and `graphify explain "<concept>"` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output.
+- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing.
+- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context.
+- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost).

apps/desktop/Scraper/AStar.Dev.Wallpaper.Scraper/Services/FileClassificationImportExportService.cs

@@ -31,17 +31,75 @@ internal async Task<Result<Unit, ScrapeError>> ImportClassificationsAsync((List<
 
     private async Task<Result<Unit, ScrapeError>> ImportLevelAsync(AppDbContext context, int level, (List<FileClassificationCategoryEntity> Categories, List<FileClassificationKeywordEntity> Keywords) classifications, CancellationToken token)
     {
-        var categoriesInLevel = classifications.Categories.Where(c => c.Level == level).OrderBy(c => c.ParentId).ThenBy(c => c.Name);
+        var categoriesInLevel = classifications.Categories.Where(c => c.Level == level).OrderBy(c => c.ParentId).ThenBy(c => c.Name).ToList();
 
-        foreach (var category in categoriesInLevel)
-            await ImportCategoryAsync(context, category, classifications.Keywords, token)
-                .TapAsync(onSuccess: _ => { }, onFailure: error => logger.Error("Failed to import classification category: {CategoryName} (Level {Level}). {ErrorMessage}", category.Name, category.Level, error.Message))
+        logger.Information("Importing {Count} categories for level {Level}", categoriesInLevel.Count, level);
+
+        return await BatchImportLevelAsync(context, categoriesInLevel, classifications.Keywords, token)
+            .OrElseAsync(_ => FallbackImportLevelAsync(context, categoriesInLevel, classifications.Keywords, token));
+    }
+
+    private async Task<Result<Unit, ScrapeError>> BatchImportLevelAsync(AppDbContext context, List<FileClassificationCategoryEntity> categories, List<FileClassificationKeywordEntity> keywords, CancellationToken token)
+        => (await Try.RunAsync(() => BatchImportLevelInternalAsync(context, categories, keywords, token)).ConfigureAwait(false))
+            .ToResult(exception => (ScrapeError)ScrapeErrorFactory.CreateRepositoryOperationFailed(nameof(BatchImportLevelAsync), exception.Message))
+            .Tap(
+                onSuccess: _ => logger.Information("Successfully batch-imported {Count} categories", categories.Count),
+                onFailure: error =>
+                {
+                    logger.Warning("Batch import failed: {ErrorMessage}. Falling back to individual processing.", error.Message);
+                    DetachUnsavedEntries(context);
+                });
+
+    private static async Task<Unit> BatchImportLevelInternalAsync(AppDbContext context, List<FileClassificationCategoryEntity> categories, List<FileClassificationKeywordEntity> keywords, CancellationToken token)
+    {
+        foreach (var category in categories)
+        {
+            var target = await FindExistingCategoryAsync(context, category, token).ConfigureAwait(false);
+
+            if (target is null)
+            {
+                target = new FileClassificationCategoryEntity
+                {
+                    Id = category.Id,
+                    Name = category.Name,
+                    Level = category.Level,
+                    ParentId = category.ParentId,
+                    IsFamous = category.IsFamous,
+                    IsInternet = category.IsInternet,
+                    IncludeInSearch = category.IncludeInSearch
+                };
+                context.FileClassificationCategories.Add(target);
+            }
+            else
+            {
+                target.IsFamous = category.IsFamous;
+                target.IsInternet = category.IsInternet;
+                target.IncludeInSearch = category.IncludeInSearch;
+            }
+
+            await AddMissingKeywordsAsync(context, target.Id, category.Id, keywords, token).ConfigureAwait(false);
+        }
+
+        await context.SaveChangesAsync(token).ConfigureAwait(false);
+
+        return Unit.Value;
+    }
+
+    private async Task<Result<Unit, ScrapeError>> FallbackImportLevelAsync(AppDbContext context, List<FileClassificationCategoryEntity> categories, List<FileClassificationKeywordEntity> keywords, CancellationToken token)
+    {
+        logger.Information("Processing {Count} categories individually with fallback handling", categories.Count);
+
+        foreach (var category in categories)
+            await ImportCategoryWithFallbackAsync(context, category, keywords, token)
+                .TapAsync(
+                    onSuccess: _ => { },
+                    onFailure: error => logger.Error("Failed to import category: {CategoryName} (Level {Level}). {ErrorMessage}", category.Name, category.Level, error.Message))
                 .ConfigureAwait(false);
 
         return Result.Success<Unit, ScrapeError>(Unit.Value);
     }
 
-    private async Task<Result<Unit, ScrapeError>> ImportCategoryAsync(AppDbContext context, FileClassificationCategoryEntity category, List<FileClassificationKeywordEntity> keywords, CancellationToken token)
+    private async Task<Result<Unit, ScrapeError>> ImportCategoryWithFallbackAsync(AppDbContext context, FileClassificationCategoryEntity category, List<FileClassificationKeywordEntity> keywords, CancellationToken token)
     {
         logger.Information("Importing classification category: {CategoryName} (Level {Level})", category.Name, category.Level);
 

apps/desktop/Scraper/AStar.Dev.Wallpaper.Scraper/Workflows/SearchWorkflow.cs

@@ -10,43 +10,41 @@ namespace AStar.Dev.Wallpaper.Scraper.Workflows;
 
 public sealed class SearchWorkflow(SearchResultsPage searchResultsPage, ScrapeConfiguration injectedScrapeConfiguration, ConfigurationSaver configurationSaver, ImagePageService imagePageService, IDirectoryHelper directoryHelper, ILogger logger, IDelayStrategy delayStrategy, TimeProvider timeProvider, PagedScrapeRunner pagedScrapeRunner)
 {
-    private SearchProgress progress = null!;
-
     public Task<Result<Unit, ScrapeError>> RunAsync(CancellationToken cancellationToken = default)
     {
-        progress = SearchProgressFactory.Create(injectedScrapeConfiguration.SearchConfiguration, injectedScrapeConfiguration.ScrapeDirectories);
+        var progress = SearchProgressFactory.Create(injectedScrapeConfiguration.SearchConfiguration, injectedScrapeConfiguration.ScrapeDirectories);
         var searchCategories = SearchProgressFunctions.FilterSearchCategories(progress.SearchConfiguration, progress.SearchConfiguration.SearchCategories);
 
-        return ProcessSearchCategoriesAsync(searchCategories, cancellationToken).LogFailure(logger);
+        return ProcessSearchCategoriesAsync(progress, searchCategories, cancellationToken).LogFailure(logger);
     }
 
-    private async Task<Result<Unit, ScrapeError>> ProcessSearchCategoriesAsync(IReadOnlyList<Category> searchCategories, CancellationToken cancellationToken)
+    private async Task<Result<Unit, ScrapeError>> ProcessSearchCategoriesAsync(SearchProgress progress, IReadOnlyList<Category> searchCategories, CancellationToken cancellationToken)
     {
         Result<Unit, ScrapeError> outcome = Unit.Value;
 
         foreach (var searchCategory in searchCategories)
         {
             cancellationToken.ThrowIfCancellationRequested();
 
-            outcome = await outcome.BindAsync(_ => ProcessSearchCategoryAsync(searchCategory, cancellationToken)).ConfigureAwait(false);
+            outcome = await outcome.BindAsync(_ => ProcessSearchCategoryAsync(progress, searchCategory, cancellationToken)).ConfigureAwait(false);
         }
 
         return outcome;
     }
 
-    private Task<Result<Unit, ScrapeError>> ProcessSearchCategoryAsync(Category searchCategory, CancellationToken cancellationToken)
+    private Task<Result<Unit, ScrapeError>> ProcessSearchCategoryAsync(SearchProgress progress, Category searchCategory, CancellationToken cancellationToken)
     {
         string combinedSearchString = $"{progress.SearchConfiguration.SearchStringPrefix}{searchCategory.Id}{progress.SearchConfiguration.SearchStringSuffix}";
-        progress = SearchProgressFunctions.UpdateSearchDetails(progress, combinedSearchString);
+        var updatedProgress = SearchProgressFunctions.UpdateSearchDetails(progress, combinedSearchString);
 
-        return searchResultsPage.LoadSearchPageAsync(combinedSearchString, progress.SearchConfiguration.StartingPageNumber)
+        return searchResultsPage.LoadSearchPageAsync(combinedSearchString, updatedProgress.SearchConfiguration.StartingPageNumber)
             .BindAsync(_ => searchResultsPage.PageInfoAsync())
-            .BindAsync(pageInfo => ProcessCategoryPageInfoAsync(searchCategory, combinedSearchString, pageInfo, cancellationToken));
+            .BindAsync(pageInfo => ProcessCategoryPageInfoAsync(updatedProgress, searchCategory, combinedSearchString, pageInfo, cancellationToken));
     }
 
-    private async Task<Result<Unit, ScrapeError>> ProcessCategoryPageInfoAsync(Category searchCategory, string combinedSearchString, PageInfo pageInfo, CancellationToken cancellationToken)
+    private async Task<Result<Unit, ScrapeError>> ProcessCategoryPageInfoAsync(SearchProgress progress, Category searchCategory, string combinedSearchString, PageInfo pageInfo, CancellationToken cancellationToken)
     {
-        progress = SearchProgressFunctions.UpdateTotalPages(progress, pageInfo.PageCount);
+        var updatedProgress = SearchProgressFunctions.UpdateTotalPages(progress, pageInfo.PageCount);
 
         if (searchCategory.IsUpToDate(pageInfo.ImageCount, pageInfo.PageCount))
         {
@@ -57,14 +55,14 @@ private async Task<Result<Unit, ScrapeError>> ProcessCategoryPageInfoAsync(Categ
         }
 
         int startingPage = searchCategory.LastPageVisited > 0 ? searchCategory.LastPageVisited : 1;
-        progress = progress with { SearchConfiguration = progress.SearchConfiguration with { StartingPageNumber = startingPage, }, };
+        updatedProgress = updatedProgress with { SearchConfiguration = updatedProgress.SearchConfiguration with { StartingPageNumber = startingPage, }, };
 
         logger.Debug("Visiting {Category} from page {StartingPage} now...", searchCategory.Name, startingPage);
-        progress = SearchProgressFunctions.UpdateSubDirectory(progress, pageInfo.SubDirectoryName);
+        updatedProgress = SearchProgressFunctions.UpdateSubDirectory(updatedProgress, pageInfo.SubDirectoryName);
 
-        _ = directoryHelper.CreateDirectoryIfRequired([progress.ScrapeDirectories.RootDirectory.CombinePath(progress.ScrapeDirectories.BaseDirectory, pageInfo.SubDirectoryName),]);
+        _ = directoryHelper.CreateDirectoryIfRequired([updatedProgress.ScrapeDirectories.RootDirectory.CombinePath(updatedProgress.ScrapeDirectories.BaseDirectory, pageInfo.SubDirectoryName),]);
 
-        return await ProcessAllCategoryPagesAsync(searchCategory, combinedSearchString, cancellationToken)
+        return await ProcessAllCategoryPagesAsync(updatedProgress, searchCategory, combinedSearchString, cancellationToken)
             .BindAsync(_ => SaveCategoryProgressAsync(searchCategory, pageInfo, cancellationToken))
             .ConfigureAwait(false);
     }
@@ -78,15 +76,15 @@ private Task<Result<Unit, ScrapeError>> SaveCategoryProgressAsync(Category searc
         return configurationSaver.SaveUpdatedConfigurationAsync(cancellationToken);
     }
 
-    private async Task<Result<Unit, ScrapeError>> ProcessAllCategoryPagesAsync(Category searchCategory, string combinedSearchString, CancellationToken cancellationToken)
+    private async Task<Result<Unit, ScrapeError>> ProcessAllCategoryPagesAsync(SearchProgress progress, Category searchCategory, string combinedSearchString, CancellationToken cancellationToken)
     {
         long startTimestamp = timeProvider.GetTimestamp();
         logger.Debug("About to visit the specific {Category} pages now...", searchCategory.Name);
 
         var plan = PagedScrapePlanFactory.Create(
             progress.SearchConfiguration.StartingPageNumber,
             progress.SearchConfiguration.TotalPages,
-            pageNumber => RecordCategoryPageProgress(searchCategory, pageNumber),
+            pageNumber => RecordCategoryPageProgress(progress, searchCategory, pageNumber),
             pageNumber => searchResultsPage.LoadSearchPageAsync(combinedSearchString, pageNumber),
             () => searchResultsPage.ImagePageLinksAsync(),
             (links, innerCt) => imagePageService.GetTheImagePagesAsync(links, searchCategory.Id, searchCategory.Name, innerCt));
@@ -95,10 +93,9 @@ private async Task<Result<Unit, ScrapeError>> ProcessAllCategoryPagesAsync(Categ
             .Tap(_ => logger.Information("Completed visiting the {Category}. Total time: {CategoryVisitDuration}", searchCategory.Name, timeProvider.GetElapsedTime(startTimestamp)));
     }
 
-    private void RecordCategoryPageProgress(Category searchCategory, int pageNumber)
+    private void RecordCategoryPageProgress(SearchProgress progress, Category searchCategory, int pageNumber)
     {
         logger.Debug("About to visit page {page} (of {totalPages}) for {Category} now...", pageNumber, progress.SearchConfiguration.TotalPages, searchCategory.Name);
-        progress = progress with { SearchConfiguration = progress.SearchConfiguration with { StartingPageNumber = pageNumber, }, };
         searchCategory.LastPageVisited = pageNumber;
     }
 }