Image processing improvements

.github/copilot-instructions.md

@@ -0,0 +1,12 @@
+# Copilot Instructions
+
+## General Guidelines
+- First general instruction
+- Second general instruction
+
+## Code Style
+- Use specific formatting rules
+- Follow naming conventions
+
+## Async Programming
+- When methods return `IAsyncEnumerable<T>`, use the `[EnumeratorCancellation]` attribute from `System.Runtime.CompilerServices` for the `CancellationToken` parameter to properly propagate cancellation through the async enumerable chain.

ImageMapper.slnx

@@ -1,6 +1,7 @@
 <Solution>
   <Folder Name="/Solution Items/">
     <File Path=".github/workflows/build.yml" />
+    <File Path="AGENTS.md" />
     <File Path="LICENSE" />
     <File Path="README.md" />
   </Folder>

README.md

@@ -2,14 +2,17 @@
 
 [![Build](https://github.com/tjmoore/image-mapper/actions/workflows/build.yml/badge.svg)](https://github.com/tjmoore/image-mapper/actions/workflows/build.yml)
 
-ImageMapper is a server hosted application that processes images from a configured server folder,
-extracts GPS metadata, and in a web application generates an interactive map displaying the locations of these images
+ImageMapper is a server hosted application that processes images from a configured server folder and maps them based on geotagged metadata within the images.
 
 This application is built using .NET and leverages the Leaflet.js library for map rendering
 
+Initial development has partly been an experiment in using AI tools to assist in code generation, architecture design, and unit test creation.
+With strict human review and modification to ensure quality and correctness.
+
 ## Dependencies
 
 - .NET 10 (likely will work with .NET 8+)
+- [Aspire](https://aspire.dev/)
 - [MetadataExtractor](https://github.com/drewnoakes/metadata-extractor-dotnet)
 - [Leaflet.js](https://leafletjs.com/)
 - [openstreetmap.org](https://www.openstreetmap.org/)
@@ -60,12 +63,12 @@ The folder for images can be configured via `appsettings.json` file in ImageMapp
 
 ## TODO
 
-- Further unit tests
-- Optimise image processing for large number of images and folders
+- Restore cluster grouping of markers when dynamically loading images
+- Show progress when loading images
+- Potential for optimising Leaflet rendering, only render markers in view etc? Would need back-end to cache image locations and support querying by bounding box?
 - Abstract file enumeration and loading to allow varied sources not just a file folder
-- Caching (minimally in-memory/redis etc, future database caching of metadata. With detection of image file changes)
+- Caching
 - Configure map tile provider options
 - UI improvements, filtering etc
-- Optimise Leaflet rendering from back-end images especially to handle large numbers of images
 - Error handling and logging improvements
 - Container support

src/ImageMapper.Api/Controllers/ImagesController.cs

@@ -12,13 +12,9 @@ public class ImagesController(IImageService svc) : ControllerBase
     private readonly IImageService _svc = svc;
 
     [HttpGet]
-    public async Task<IEnumerable<ImageInfo>> Get(CancellationToken ct)
+    public IAsyncEnumerable<ImageInfo> Get(CancellationToken ct)
     {
-        var images = await _svc.GetImagesAsync(ct);
-
-        Log.Debug("GET /api/images - Retrieved {Count} images", images.Count());
-
-        return images;
+        return _svc.GetImagesAsync(ct);
     }
 
     [HttpGet("raw/{**relativePath}")]

src/ImageMapper.Api/Services/IImageService.cs

@@ -4,6 +4,6 @@ namespace ImageMapper.Api.Services;
 
 public interface IImageService
 {
-    Task<IEnumerable<ImageInfo>> GetImagesAsync(CancellationToken ct = default);
+    IAsyncEnumerable<ImageInfo> GetImagesAsync(CancellationToken ct = default);
     Task<byte[]?> GetImageBytesAsync(string relativePath, CancellationToken ct = default);
 }

src/ImageMapper.Api/Services/ImageService.cs

@@ -2,6 +2,7 @@
 using MetadataExtractor;
 using MetadataExtractor.Formats.Exif;
 using Serilog;
+using System.Runtime.CompilerServices;
 
 namespace ImageMapper.Api.Services;
 
@@ -18,19 +19,16 @@ public ImageService(IConfiguration config)
         Log.Information("ImageService initialized with ImageFolder: {ImageFolder}", _imagesRoot);
     }
 
-    public async Task<IEnumerable<ImageInfo>> GetImagesAsync(CancellationToken ct = default)
+    public async IAsyncEnumerable<ImageInfo> GetImagesAsync([EnumeratorCancellation] CancellationToken ct = default)
     {
         if (!System.IO.Directory.Exists(_imagesRoot))
-            return [];
-
-        // TODO: this will need to be optimised for large file stores and possibly return async enumerable
+            yield break;
 
         var extensions = new[] { ".jpg", ".jpeg", ".png", ".tif", ".tiff", ".nef" };
 
         var files = System.IO.Directory.EnumerateFiles(_imagesRoot, "*.*", SearchOption.AllDirectories)
             .Where(f => extensions.Contains(Path.GetExtension(f).ToLowerInvariant()));
 
-        var list = new List<ImageInfo>();
         foreach (string f in files)
         {
             ct.ThrowIfCancellationRequested();
@@ -57,10 +55,8 @@ public async Task<IEnumerable<ImageInfo>> GetImagesAsync(CancellationToken ct =
             {
                 Log.Warning("Failed to read metadata for file: {File}", f);
             }
-            list.Add(info);
+            yield return info;
         }
-
-        return await Task.FromResult(list);
     }
 
     public async Task<byte[]?> GetImageBytesAsync(string relativePath, CancellationToken ct = default)

src/ImageMapper.Tests/ImageMapper.Tests.csproj

@@ -25,6 +25,7 @@
   <ItemGroup>
     <ProjectReference Include="..\ImageMapper.Api\ImageMapper.Api.csproj" />
     <ProjectReference Include="..\ImageMapper.AppHost\ImageMapper.AppHost.csproj" />
+    <ProjectReference Include="..\ImageMapper.Web\ImageMapper.Web.csproj" />
   </ItemGroup>
 
   <ItemGroup>

src/ImageMapper.Tests/ImagesApiTest.cs

@@ -1,4 +1,5 @@
 using ImageMapper.Api.Services;
+using ImageMapper.Models;
 using Microsoft.Extensions.Configuration;
 
 namespace ImageMapper.Tests
@@ -133,5 +134,123 @@ public async Task GetImageBytesAsyncReturnsNullForNonExistentFilesInSubfolders(s
             // Assert
             Assert.That(bytes, Is.Null);
         }
+
+        [Test]
+        public async Task GetImagesAsyncReturnsAllImagesFromDirectory()
+        {
+            // Arrange
+            var config = new ConfigurationBuilder()
+                .AddInMemoryCollection(new Dictionary<string, string?> { { "ImageFolder", _testImagesDirectory } })
+                .Build();
+            var service = new ImageService(config);
+
+            // Act
+            var images = new List<ImageInfo>();
+            await foreach (var image in service.GetImagesAsync())
+            {
+                images.Add(image);
+            }
+
+            // Assert
+            Assert.That(images, Has.Count.EqualTo(3));
+            Assert.That(images.Select(i => i.FileName), Does.Contain("test-image.jpg"));
+            Assert.That(images.Select(i => i.FileName), Does.Contain("nested-image.jpg"));
+            Assert.That(images.Select(i => i.FileName), Does.Contain("deep-image.png"));
+        }
+
+        [Test]
+        public async Task GetImagesAsyncReturnsCorrectRelativePaths()
+        {
+            // Arrange
+            var config = new ConfigurationBuilder()
+                .AddInMemoryCollection(new Dictionary<string, string?> { { "ImageFolder", _testImagesDirectory } })
+                .Build();
+            var service = new ImageService(config);
+
+            // Act
+            var images = new List<ImageInfo>();
+            await foreach (var image in service.GetImagesAsync())
+            {
+                images.Add(image);
+            }
+
+            // Assert
+            Assert.That(images.Select(i => i.RelativePath), Does.Contain("test-image.jpg"));
+            Assert.That(images.Select(i => i.RelativePath), Does.Contain("subfolder/nested-image.jpg"));
+            Assert.That(images.Select(i => i.RelativePath), Does.Contain("subfolder/deep/deep-image.png"));
+        }
+
+        [Test]
+        public async Task GetImagesAsyncReturnsEmptyListForNonExistentDirectory()
+        {
+            // Arrange
+            var config = new ConfigurationBuilder()
+                .AddInMemoryCollection(new Dictionary<string, string?> { { "ImageFolder", Path.Combine(_testImagesDirectory, "nonexistent") } })
+                .Build();
+            var service = new ImageService(config);
+
+            // Act
+            var images = new List<ImageInfo>();
+            await foreach (var image in service.GetImagesAsync())
+            {
+                images.Add(image);
+            }
+
+            // Assert
+            Assert.That(images, Is.Empty);
+        }
+
+        [Test]
+        public async Task GetImagesAsyncIsCancellable()
+        {
+            // Arrange
+            var config = new ConfigurationBuilder()
+                .AddInMemoryCollection(new Dictionary<string, string?> { { "ImageFolder", _testImagesDirectory } })
+                .Build();
+            var service = new ImageService(config);
+            using var cts = new CancellationTokenSource();
+            cts.Cancel();
+
+            // Act & Assert
+            var ex = Assert.ThrowsAsync<OperationCanceledException>(
+                async () =>
+                {
+                    await foreach (var image in service.GetImagesAsync(cts.Token))
+                    {
+                        // This should throw due to cancellation
+                    }
+                });
+
+            Assert.That(ex, Is.Not.Null);
+        }
+
+        [Test]
+        public async Task GetImagesAsyncFiltersOnlyImageExtensions()
+        {
+            // Arrange
+            var config = new ConfigurationBuilder()
+                .AddInMemoryCollection(new Dictionary<string, string?> { { "ImageFolder", _testImagesDirectory } })
+                .Build();
+
+            // Create a non-image file
+            var nonImagePath = Path.Combine(_testImagesDirectory, "readme.txt");
+            File.WriteAllText(nonImagePath, "This is not an image");
+
+            var service = new ImageService(config);
+
+            // Act
+            var images = new List<ImageInfo>();
+            await foreach (var image in service.GetImagesAsync())
+            {
+                images.Add(image);
+            }
+
+            // Assert
+            Assert.That(images, Has.Count.EqualTo(3)); // Still only 3 images, not 4
+            Assert.That(images.Select(i => i.FileName), Does.Not.Contain("readme.txt"));
+
+            // Cleanup
+            File.Delete(nonImagePath);
+        }
     }
 }

src/ImageMapper.Tests/IntegrationTest.cs

@@ -1,4 +1,6 @@
 using Aspire.Hosting;
+using ImageMapper.Models;
+using ImageMapper.Web.Client;
 using Microsoft.Extensions.Logging;
 using Serilog;
 
@@ -8,6 +10,7 @@ public class IntegrationTest
     {
         private static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(30);
         private string _testImagesDirectory = null!;
+        private string _testSubdirectory = null!;
 
         [OneTimeSetUp]
         public void OneTimeSetUp()
@@ -18,6 +21,13 @@ public void OneTimeSetUp()
             // Create a test image file
             var testImagePath = Path.Combine(_testImagesDirectory, "test-image.jpg");
             File.WriteAllBytes(testImagePath, [0xFF, 0xD8, 0xFF, 0xE0]); // JPEG magic bytes
+
+            // Create a subdirectory with test images
+            _testSubdirectory = Path.Combine(_testImagesDirectory, "subfolder");
+            Directory.CreateDirectory(_testSubdirectory);
+
+            var subImagePath = Path.Combine(_testSubdirectory, "nested-image.png");
+            File.WriteAllBytes(subImagePath, [0x89, 0x50, 0x4E, 0x47]); // PNG magic bytes
         }
 
         [OneTimeTearDown]
@@ -75,6 +85,95 @@ public async Task GetWebResourceRootReturnsOkStatusCode()
             Assert.That(response.StatusCode, Is.EqualTo(HttpStatusCode.OK));
         }
 
+        [Test]
+        public async Task GetImagesListReturnsAllImages()
+        {
+            // Arrange
+            using var cts = new CancellationTokenSource(DefaultTimeout);
+            var cancellationToken = cts.Token;
+
+            await using var app = await BuildAndStartAppAsync(cancellationToken, _testImagesDirectory);
+            using var httpClient = app.CreateHttpClient("imagemapper-api");
+            await app.ResourceNotifications.WaitForResourceHealthyAsync("imagemapper-web", cancellationToken).WaitAsync(DefaultTimeout, cancellationToken);
+
+            var fetcher = new ImageItemFetcher(httpClient);
+
+            // Act
+            var images = new List<ImageInfo>();
+            await foreach (var image in fetcher.Fetch(cancellationToken))
+            {
+                if (image != null)
+                    images.Add(image);
+            }
+
+            // Assert
+            Assert.That(images, Has.Count.EqualTo(2));
+            Assert.That(images.Select(i => i.FileName), Does.Contain("test-image.jpg"));
+            Assert.That(images.Select(i => i.FileName), Does.Contain("nested-image.png"));
+        }
+
+        [Test]
+        public async Task GetImagesListReturnsCorrectRelativePaths()
+        {
+            // Arrange
+            using var cts = new CancellationTokenSource(DefaultTimeout);
+            var cancellationToken = cts.Token;
+
+            await using var app = await BuildAndStartAppAsync(cancellationToken, _testImagesDirectory);
+            using var httpClient = app.CreateHttpClient("imagemapper-api");
+            await app.ResourceNotifications.WaitForResourceHealthyAsync("imagemapper-web", cancellationToken).WaitAsync(DefaultTimeout, cancellationToken);
+
+            var fetcher = new ImageItemFetcher(httpClient);
+
+            // Act
+            var images = new List<ImageInfo>();
+            await foreach (var image in fetcher.Fetch(cancellationToken))
+            {
+                if (image != null)
+                    images.Add(image);
+            }
+
+            // Assert
+            Assert.That(images.Select(i => i.RelativePath), Does.Contain("test-image.jpg"));
+            Assert.That(images.Select(i => i.RelativePath), Does.Contain("subfolder/nested-image.png"));
+        }
+
+        [Test]
+        public async Task GetImagesListReturnsEmptyWhenNoImagesExist()
+        {
+            // Arrange
+            using var cts = new CancellationTokenSource(DefaultTimeout);
+            var cancellationToken = cts.Token;
+
+            var emptyDirectory = Path.Combine(TestContext.CurrentContext.TestDirectory, $"empty-test-{Guid.NewGuid()}");
+            Directory.CreateDirectory(emptyDirectory);
+
+            try
+            {
+                await using var app = await BuildAndStartAppAsync(cancellationToken, emptyDirectory);
+                using var httpClient = app.CreateHttpClient("imagemapper-api");
+                await app.ResourceNotifications.WaitForResourceHealthyAsync("imagemapper-web", cancellationToken).WaitAsync(DefaultTimeout, cancellationToken);
+
+                var fetcher = new ImageItemFetcher(httpClient);
+
+                // Act
+                var images = new List<ImageInfo>();
+                await foreach (var image in fetcher.Fetch(cancellationToken))
+                {
+                    if (image != null)
+                        images.Add(image);
+                }
+
+                // Assert
+                Assert.That(images, Is.Empty);
+            }
+            finally
+            {
+                if (Directory.Exists(emptyDirectory))
+                    Directory.Delete(emptyDirectory, recursive: true);
+            }
+        }
+
         [Test]
         public async Task GetRawImageReturnsExistingFile()
         {