feat: Enhance product identity resolution and category metadata for smartphones

Author: hutchpd

ProductNormaliser.Application.Tests/IdentityAndMergeTests.cs

@@ -65,8 +65,23 @@ public void GetAll_EnablesSupportedAndExperimentalActiveCategories()
         Assert.Multiple(() =>
         {
             Assert.That(enabledCategories.Select(category => category.CategoryKey), Is.EqualTo(new[] { "tv", "monitor", "laptop", "tablet", "smartphone", "headphones", "speakers" }));
-            Assert.That(enabledCategories.Count(category => category.CrawlSupportStatus == CrawlSupportStatus.Supported), Is.EqualTo(3));
-            Assert.That(enabledCategories.Count(category => category.CrawlSupportStatus == CrawlSupportStatus.Experimental), Is.EqualTo(4));
+            Assert.That(enabledCategories.Count(category => category.CrawlSupportStatus == CrawlSupportStatus.Supported), Is.EqualTo(4));
+            Assert.That(enabledCategories.Count(category => category.CrawlSupportStatus == CrawlSupportStatus.Experimental), Is.EqualTo(3));
+        });
+    }
+
+    [Test]
+    public void GetByKey_ReturnsSmartphoneAsSupportedEnabledCategory()
+    {
+        var category = DefaultCategoryMetadataCatalog.GetByKey("smartphone");
+
+        Assert.That(category, Is.Not.Null);
+        Assert.Multiple(() =>
+        {
+            Assert.That(category!.DisplayName, Is.EqualTo("Smartphones"));
+            Assert.That(category.CrawlSupportStatus, Is.EqualTo(CrawlSupportStatus.Supported));
+            Assert.That(category.SchemaCompletenessScore, Is.EqualTo(0.91m));
+            Assert.That(category.IsEnabled, Is.True);
         });
     }
 }

ProductNormaliser.Domain.Tests/DefaultCategoryMetadataCatalogTests.cs

@@ -10,6 +10,15 @@ public sealed class ProductIdentityResolver(
     ConfidenceScorer? confidenceScorer = null,
     ICategoryAttributeNormaliserRegistry? categoryAttributeNormaliserRegistry = null) : IProductIdentityResolver
 {
+    private const string SmartphoneCategoryKey = "smartphone";
+    private static readonly IReadOnlyDictionary<string, string[]> StrongDisambiguatorKeysByCategory = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase)
+    {
+        [SmartphoneCategoryKey] = ["storage_capacity_gb", "manufacturer_part_number", "regional_variant", "carrier_lock_status"],
+        ["tablet"] = ["connectivity", "cellular_generation", "manufacturer_part_number", "regional_variant", "storage_capacity_gb"],
+        ["headphones"] = ["connection_type", "manufacturer_part_number"],
+        ["speakers"] = ["connection_type", "speaker_type", "manufacturer_part_number"]
+    };
+
     private readonly ProductFingerprintBuilder fingerprintBuilder = fingerprintBuilder ?? new ProductFingerprintBuilder();
     private readonly ConfidenceScorer confidenceScorer = confidenceScorer ?? new ConfidenceScorer();
     private readonly ICategoryAttributeNormaliserRegistry categoryAttributeNormaliserRegistry = categoryAttributeNormaliserRegistry ?? DefaultCategoryRegistries.CreateAttributeNormaliserRegistry();
@@ -46,7 +55,15 @@ public ProductIdentityMatchResult Match(SourceProduct sourceProduct, IReadOnlyCo
             };
         }
 
-        var brandModelMatch = candidates.FirstOrDefault(candidate =>
+        var manufacturerPartNumberMatch = FindManufacturerPartNumberMatch(sourceProduct, candidates);
+        if (manufacturerPartNumberMatch is not null)
+        {
+            return manufacturerPartNumberMatch;
+        }
+
+        var broaderCandidates = FilterCandidatesForBroaderMatching(sourceProduct, candidates);
+
+        var brandModelMatch = broaderCandidates.FirstOrDefault(candidate =>
             string.Equals(candidate.Brand?.Trim(), sourceProduct.Brand?.Trim(), StringComparison.OrdinalIgnoreCase)
             && string.Equals(candidate.ModelNumber?.Trim(), sourceProduct.ModelNumber?.Trim(), StringComparison.OrdinalIgnoreCase)
             && !string.IsNullOrWhiteSpace(candidate.Brand)
@@ -63,13 +80,13 @@ public ProductIdentityMatchResult Match(SourceProduct sourceProduct, IReadOnlyCo
             };
         }
 
-        var categoryIdentityMatch = FindCategoryIdentityMatch(sourceProduct, candidates);
+        var categoryIdentityMatch = FindCategoryIdentityMatch(sourceProduct, broaderCandidates);
         if (categoryIdentityMatch is not null)
         {
             return categoryIdentityMatch;
         }
 
-        var bestSimilarityMatch = candidates
+        var bestSimilarityMatch = broaderCandidates
             .Select(candidate => new
             {
                 Candidate = candidate,
@@ -89,6 +106,17 @@ public ProductIdentityMatchResult Match(SourceProduct sourceProduct, IReadOnlyCo
             };
         }
 
+        var strongConflictReason = GetStrongVariantConflictReason(sourceProduct, candidates);
+        if (strongConflictReason is not null)
+        {
+            return new ProductIdentityMatchResult
+            {
+                IsMatch = false,
+                Confidence = 0.00m,
+                MatchReason = strongConflictReason
+            };
+        }
+
         return new ProductIdentityMatchResult
         {
             IsMatch = false,
@@ -97,6 +125,31 @@ public ProductIdentityMatchResult Match(SourceProduct sourceProduct, IReadOnlyCo
         };
     }
 
+    private ProductIdentityMatchResult? FindManufacturerPartNumberMatch(SourceProduct sourceProduct, IReadOnlyCollection<CanonicalProduct> candidates)
+    {
+        if (!TryGetStrongDisambiguatorKeys(sourceProduct.CategoryKey, out _)
+            || !TryGetComparableValue(sourceProduct, "manufacturer_part_number", out var sourceManufacturerPartNumber))
+        {
+            return null;
+        }
+
+        var match = candidates.FirstOrDefault(candidate =>
+            string.Equals(candidate.CategoryKey, sourceProduct.CategoryKey, StringComparison.OrdinalIgnoreCase)
+            && TryGetComparableValue(candidate, "manufacturer_part_number", out var candidateManufacturerPartNumber)
+            && string.Equals(candidateManufacturerPartNumber, sourceManufacturerPartNumber, StringComparison.OrdinalIgnoreCase)
+            && PrimaryIdentitySignalsAreCompatible(sourceProduct, candidate));
+
+        return match is null
+            ? null
+            : new ProductIdentityMatchResult
+            {
+                CanonicalProductId = match.Id,
+                IsMatch = true,
+                Confidence = 0.98m,
+                MatchReason = "Exact manufacturer part number match."
+            };
+    }
+
     private ProductIdentityMatchResult? FindCategoryIdentityMatch(SourceProduct sourceProduct, IReadOnlyCollection<CanonicalProduct> candidates)
     {
         var identityKeys = categoryAttributeNormaliserRegistry
@@ -138,6 +191,70 @@ public ProductIdentityMatchResult Match(SourceProduct sourceProduct, IReadOnlyCo
         };
     }
 
+    private IReadOnlyCollection<CanonicalProduct> FilterCandidatesForBroaderMatching(SourceProduct sourceProduct, IReadOnlyCollection<CanonicalProduct> candidates)
+    {
+        if (!TryGetStrongDisambiguatorKeys(sourceProduct.CategoryKey, out _))
+        {
+            return candidates;
+        }
+
+        return candidates
+            .Where(candidate => !HasStrongVariantConflict(sourceProduct, candidate))
+            .ToArray();
+    }
+
+    private static bool PrimaryIdentitySignalsAreCompatible(SourceProduct sourceProduct, CanonicalProduct candidate)
+    {
+        return !HasComparableConflict(sourceProduct, candidate, "brand")
+            && !HasComparableConflict(sourceProduct, candidate, "model_number");
+    }
+
+    private static bool HasStrongVariantConflict(SourceProduct sourceProduct, CanonicalProduct candidate)
+    {
+        if (!TryGetStrongDisambiguatorKeys(sourceProduct.CategoryKey, out var disambiguatorKeys)
+            || !string.Equals(candidate.CategoryKey, sourceProduct.CategoryKey, StringComparison.OrdinalIgnoreCase))
+        {
+            return false;
+        }
+
+        return disambiguatorKeys.Any(key => HasComparableConflict(sourceProduct, candidate, key));
+    }
+
+    private static string? GetStrongVariantConflictReason(SourceProduct sourceProduct, IReadOnlyCollection<CanonicalProduct> candidates)
+    {
+        if (!TryGetStrongDisambiguatorKeys(sourceProduct.CategoryKey, out _)
+            || !candidates.Any(candidate => HasStrongVariantConflict(sourceProduct, candidate)))
+        {
+            return null;
+        }
+
+        return sourceProduct.CategoryKey.ToLowerInvariant() switch
+        {
+            SmartphoneCategoryKey => "Strong smartphone variant conflict prevented a safe match.",
+            "tablet" => "Strong tablet variant conflict prevented a safe match.",
+            "headphones" => "Strong headphones variant conflict prevented a safe match.",
+            "speakers" => "Strong speakers variant conflict prevented a safe match.",
+            _ => "Strong category variant conflict prevented a safe match."
+        };
+    }
+
+    private static bool HasComparableConflict(SourceProduct sourceProduct, CanonicalProduct candidate, string key)
+    {
+        return TryGetComparableValue(sourceProduct, key, out var sourceValue)
+            && TryGetComparableValue(candidate, key, out var candidateValue)
+            && !string.Equals(sourceValue, candidateValue, StringComparison.OrdinalIgnoreCase);
+    }
+
+    private static bool IsSmartphoneCategory(string categoryKey)
+    {
+        return string.Equals(categoryKey, SmartphoneCategoryKey, StringComparison.OrdinalIgnoreCase);
+    }
+
+    private static bool TryGetStrongDisambiguatorKeys(string categoryKey, out string[] keys)
+    {
+        return StrongDisambiguatorKeysByCategory.TryGetValue(categoryKey, out keys!);
+    }
+
     private static int CountMatchingIdentityAttributes(SourceProduct sourceProduct, CanonicalProduct candidate, IEnumerable<string> identityKeys)
     {
         return identityKeys.Count(key => TryGetComparableValue(sourceProduct, key, out var sourceValue)

ProductNormaliser.Domain/Merging/ProductIdentityResolver.cs

@@ -71,7 +71,7 @@ The goal is not just to pick a winner, but to pick a winner with an audit trail.
 
 ## Category schema
 
-The first fully modelled category is `tv` in the TV schema provider. Additional category providers for `monitor`, `laptop`, and `refrigerator` now exist so the rest of the platform can score completeness, route normalisation, and expose dashboard metadata without assuming TV-only semantics.
+The first fully modelled category is `tv` in the TV schema provider. Additional category providers now exist for `monitor`, `laptop`, `smartphone`, `tablet`, `headphones`, `speakers`, and `refrigerator` so the rest of the platform can score completeness, route normalisation, and expose dashboard metadata without assuming TV-only semantics.
 
 The TV schema still defines the richest required and optional canonical attributes such as:
 
@@ -91,6 +91,8 @@ Adding a new category typically means:
 3. deciding identity heuristics and attribute reliability rules for that category
 4. registering metadata so the admin API and web dashboard can discover the category safely
 
+At the current maturity line, `tv`, `monitor`, `laptop`, and `smartphone` are the supported categories with the strongest schema, normalisation, and identity coverage. `tablet`, `headphones`, and `speakers` are enabled experimental categories with broader canonical field sets and category-specific normalisers, but they are still being hardened before promotion.
+
 ## How other projects use Domain
 
 - Infrastructure implements many of the Domain interfaces and persists Domain models.

ProductNormaliser.Domain/README.md

@@ -10,7 +10,7 @@ public static class DefaultCategoryMetadataCatalog
         Create("monitor", "Monitors", "display", "Display", "monitor", CrawlSupportStatus.Supported, 0.89m, true),
         Create("laptop", "Laptops", "computing", "Computing", "laptop", CrawlSupportStatus.Supported, 0.87m, true),
         Create("tablet", "Tablets", "mobile", "Mobile", "tablet", CrawlSupportStatus.Experimental, 0.78m, true),
-        Create("smartphone", "Smartphones", "mobile", "Mobile", "smartphone", CrawlSupportStatus.Experimental, 0.79m, true),
+        Create("smartphone", "Smartphones", "mobile", "Mobile", "smartphone", CrawlSupportStatus.Supported, 0.91m, true),
         Create("headphones", "Headphones", "audio", "Audio", "headphones", CrawlSupportStatus.Experimental, 0.74m, true),
         Create("speakers", "Speakers", "audio", "Audio", "speaker", CrawlSupportStatus.Experimental, 0.72m, true),
         Create("refrigerator", "Refrigerators", "kitchen_appliances", "Kitchen Appliances", "refrigerator", CrawlSupportStatus.Planned, 0.10m, false),

ProductNormaliser.Domain/Schemas/DefaultCategoryMetadataCatalog.cs

@@ -91,7 +91,7 @@ public async Task ExplorerPage_RendersEnabledSupportedAndExperimentalCategoryOpt
                 new CategoryMetadataDto { CategoryKey = "monitor", DisplayName = "Monitors", IsEnabled = true, CrawlSupportStatus = "Supported" },
                 new CategoryMetadataDto { CategoryKey = "laptop", DisplayName = "Laptops", IsEnabled = true, CrawlSupportStatus = "Supported" },
                 new CategoryMetadataDto { CategoryKey = "tablet", DisplayName = "Tablets", IsEnabled = true, CrawlSupportStatus = "Experimental" },
-                new CategoryMetadataDto { CategoryKey = "smartphone", DisplayName = "Smartphones", IsEnabled = true, CrawlSupportStatus = "Experimental" },
+                new CategoryMetadataDto { CategoryKey = "smartphone", DisplayName = "Smartphones", IsEnabled = true, CrawlSupportStatus = "Supported" },
                 new CategoryMetadataDto { CategoryKey = "headphones", DisplayName = "Headphones", IsEnabled = true, CrawlSupportStatus = "Experimental" },
                 new CategoryMetadataDto { CategoryKey = "speakers", DisplayName = "Speakers", IsEnabled = true, CrawlSupportStatus = "Experimental" },
                 new CategoryMetadataDto { CategoryKey = "refrigerator", DisplayName = "Refrigerators", IsEnabled = false, CrawlSupportStatus = "Planned" }

ProductNormaliser.Web.Tests/ProductPageRenderingTests.cs

@@ -9,7 +9,8 @@ The web host currently delivers:
 - an operator landing page that keeps the active category context visible
 - an operator landing page operational health panel for queue depth, retry backlog, recent failures, at-risk sources, and category pressure
 - an operator landing page boot-and-populate panel showing boot-ready sources, categories in context, estimated discovery seeds, and recent confirmed-product throughput
-- category selection for the rollout set: TVs, Monitors, and Laptops
+- category selection for the current supported set: TVs, Monitors, Laptops, and Smartphones
+- category selectors and explorer filters that continue to expose enabled experimental categories such as Tablets, Headphones, and Speakers with their maturity badges intact
 - seeded crawl launch and crawl-job monitoring with discovery and product progress shown together
 - canonical product exploration with quality-aware filters and paging
 - product detail pages with source comparison, evidence, conflicts, and history

ProductNormaliser.Web/README.md

@@ -2,7 +2,7 @@
 
 ProductNormaliser is an open product-intelligence engine for turning messy retail and manufacturer page data into clean, canonical, comparable product records. It crawls source pages, extracts structured product evidence, normalises attributes into a category schema, resolves identity across sources, merges competing claims into a canonical product, and keeps learning over time from quality history, disagreement patterns, and page volatility.
 
-Milestone 1 is centered on an end-to-end operator workflow for three rollout categories: `tv`, `monitor`, and `laptop`. The platform still keeps category and normalisation extension points broad enough for future electrical-goods expansion, but the completed milestone scope is the crawl, management, product, and quality experience for those three categories.
+Milestone 1 is centered on an end-to-end operator workflow for four supported categories: `tv`, `monitor`, `laptop`, and `smartphone`. The platform still keeps category and normalisation extension points broad enough for broader electrical-goods expansion, while `tablet`, `headphones`, and `speakers` remain enabled experimental categories that share the same workflow surface.
 
 ## What problem this solves
 
@@ -76,7 +76,7 @@ The solution now contains ten projects:
 
 ## Architecture at a glance
 
-1. Operators register or enable managed crawl sources and assign categories such as `tv`, `monitor`, and `laptop`.
+1. Operators register or enable managed crawl sources and assign categories such as `tv`, `monitor`, `laptop`, and `smartphone`.
 2. Each source carries a discovery profile with category entry pages, sitemap hints, allow or deny path rules, URL patterns, depth limits, and per-run budgets.
 3. A category crawl job now seeds deterministic discovery from eligible managed sources instead of relying only on pre-known targets.
 4. The discovery worker fetches sitemaps and listing pages while respecting robots rules, source throttling, depth limits, and URL budgets.
@@ -93,8 +93,8 @@ The solution now contains ten projects:
 The solution currently includes:
 
 - category metadata and schema discovery for electrical-goods families
-- category registry support for the Milestone 1 rollout set: TVs, Monitors, and Laptops
-- schema-driven attribute normalisation with category-specific providers for TVs, Monitors, and Laptops
+- category registry support for the current supported set: TVs, Monitors, Laptops, and Smartphones
+- schema-driven attribute normalisation with category-specific providers for TVs, Monitors, Laptops, Smartphones, and enabled experimental next-wave categories
 - alias handling and measurement parsing
 - structured data extraction from HTML and JSON-LD
 - MongoDB persistence for source and canonical records