From 8d5b7d667fff8bce92e66e8d6188dedb27120860 Mon Sep 17 00:00:00 2001 From: Mikko Numminen Date: Wed, 15 Jul 2026 05:11:55 +0300 Subject: [PATCH 1/2] feat(retail): graded, density-gated vulgarity forces asiaton (ADR-0039) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A vulgar pile-up ("Paskapillupersepornolehtipaviaani") was stored category=muu (rated catch-all), severity=high — Poro recognized the offensiveness (theme= offensive_language) but filed it under the rated catch-all, and the only deterministic conduct override today is the rasismi ethnic-slur lexicon, which this word doesn't hit. The demoted-category suppression (ADR-0032/0033/0038) can't help content that was never demoted. Add a deterministic, DENSITY-gated graded vulgarity scorer that forces the demoted conduct category (asiaton), a sibling to the alert (ADR-0027) and category-keyword (ADR-0036) layers: - domains/retail/vulgarity-lexicon.json — tiered Finnish profanity stems (mild/ strong) + thresholds; optional domain data, validated at boot (a demoteToCategory that isn't a declared DEMOTED category fails the boot). - VulgarityScorer (Core): level 0-3. Demote (level 3) iff the vulgar-character share >= demoteRatio AND >= demoteMinDistinctStems DISTINCT stems. Both gates matter: distinct-count (never raw occurrences) keeps a repeated single swear rated; the ratio keeps a furious-but-substantive complaint (mostly real words) rated while a nonsense pile-up (mostly stems) is demoted. - CategoryOverrideResolver hook: after the rasismi alert override and an existing demoted choice, before the product category-keyword override. Ethnic slurs stay in the alert lexicon (single hit -> rasismi); this scorer never touches them. - Wired into ingest, restructure, and the /interpret preview so the forced category is identical everywhere. "Paskapillupersepornolehtipaviaani" (3 distinct, ~0.45 share) -> asiaton; "Möivät paskaa." (1 distinct, ~0.31 share) and a substantive complaint carrying three swears (low share) stay rated. Thresholds are empirical and config-tunable — the ADR states plainly they must be measured on real examples and will mis-fire outside the fixture until tuned. The visible per-item vulgarity recognition tag for rated level 1-2 items is a deferred follow-up (Phase 2), noted in the ADR. Tests: VulgarityScorer (incl. the furious-but-substantive precision case), VulgarityLexiconSet boot validation, and CategoryOverrideResolver precedence (racism > vulgarity > product-keyword). Full solution green. --- .../0039-graded-vulgarity-density-asiaton.md | 74 ++++++++++++++ docs/decisions/README.md | 1 + domains/retail/vulgarity-lexicon.json | 11 +++ .../Ingest/IngestService.cs | 7 +- src/FeedbackIntelligence.Api/Program.cs | 14 ++- .../Structuring/VulgarityLexiconSet.cs | 78 +++++++++++++++ .../Domain/ActiveDomain.cs | 8 ++ .../Structuring/CategoryOverrideResolver.cs | 13 ++- .../Structuring/VulgarityScorer.cs | 98 +++++++++++++++++++ .../CategoryOverrideResolverTests.cs | 58 +++++++++-- .../ChannelIsolationTests.cs | 1 + .../IngestServiceTests.cs | 4 + .../TestDomains.cs | 9 ++ .../VulgarityLexiconSetTests.cs | 58 +++++++++++ .../VulgarityScorerTests.cs | 66 +++++++++++++ .../LlmStructuringServiceTests.cs | 1 + 16 files changed, 486 insertions(+), 15 deletions(-) create mode 100644 docs/decisions/0039-graded-vulgarity-density-asiaton.md create mode 100644 domains/retail/vulgarity-lexicon.json create mode 100644 src/FeedbackIntelligence.Api/Structuring/VulgarityLexiconSet.cs create mode 100644 src/FeedbackIntelligence.Core/Structuring/VulgarityScorer.cs create mode 100644 tests/FeedbackIntelligence.Api.Tests/VulgarityLexiconSetTests.cs create mode 100644 tests/FeedbackIntelligence.Api.Tests/VulgarityScorerTests.cs diff --git a/docs/decisions/0039-graded-vulgarity-density-asiaton.md b/docs/decisions/0039-graded-vulgarity-density-asiaton.md new file mode 100644 index 0000000..bd6838a --- /dev/null +++ b/docs/decisions/0039-graded-vulgarity-density-asiaton.md @@ -0,0 +1,74 @@ +# ADR-0039 — Graded vulgarity: dense, non-substantive profanity is forced to `asiaton` + +- **Status:** Accepted (2026-07-15) +- **Deciders:** Mikko +- **Follows:** [ADR-0027](0027-racism-recognition-alert-lexicon.md) (a lexicon forces a + conduct category on a single hit — for ethnic slurs → `rasismi`), + [ADR-0032](0032-unrated-nonsubstantive-categories.md) (demoted = unrated), + [ADR-0036](0036-deterministic-category-keyword-override.md) (deterministic + category-keyword override architecture) + +## Context + +A vulgar pile-up — `Paskapillupersepornolehtipaviaani` — was stored `category=muu` +(the rated catch-all), `severity=high`, `theme=offensive_language`. Poro *recognized* +the offensiveness (the theme) but filed it under the rated catch-all, not `asiaton`, +and rated nonsense `high`. Poro ignoring the `asiaton` affordance is the exact ADR-0032 +limitation ("the model ignores the affordance"); the demoted-category suppression +(ADR-0032/0033/0038) can't help, because the item was never in a demoted category. The +only deterministic conduct-forcing rule today is the rasismi ethnic-slur lexicon +(ADR-0027) — this word is not ethnic, so nothing fired. + +The owner's requirement, and its crucial nuance: **a lone swear inside real feedback** +(`Möivät paskaa` — "they sold crap", a crude but genuine quality complaint) **must stay +rated**; a **pile-up of distinct vulgar stems mashed into nonsense** is not feedback and +belongs in the moderation view. ADR-0032 gave up on *generic* garbage detection because +Poro routed keyboard-mash unpredictably — but dense vulgarity has a **deterministic +signal** (the stems themselves), so it is catchable where generic gibberish is not. + +## Decision + +**A deterministic, DENSITY-gated graded vulgarity scorer forces the demoted conduct +category (`asiaton`).** + +1. **Domain lexicon** (`domains//vulgarity-lexicon.json`, optional like the + category-keyword lexicon): Finnish profanity **stems in tiers** (`mild`, `strong`), + plus the demote thresholds. Owner-authored (register is cultural), matched as + case-insensitive invariant substrings — the same contract as the alert lexicon. +2. **Graded level 0–3** per message (`VulgarityScorer`): + - **0** none · **1** incidental (a mild stem) · **2** notable (a strong stem) — + Levels 1–2 stay **rated** (a Phase-2 `⚑` recognition tag will read them; see below). + - **3** dominant → **force `asiaton`** (demoted → unrated, count-only, moderation view). +3. **Density gate for Level 3** — demote **iff** the vulgar-character share of the message + is `>= demoteRatio` **AND** there are `>= demoteMinDistinctStems` **distinct** stems. + Both conditions matter: + - *distinct-count* (not raw occurrences): repeating one swear (`paska`… `paska`) does + not inflate it — a lone stem stays rated (the owner's "singletons shouldn't flag"). + - *ratio*: a furious-but-real complaint carrying three swears in a substantive sentence + is mostly real words (low ratio) → **stays rated**; a nonsense pile-up is mostly + vulgar stems (high ratio) → demoted. The two conditions together separate *angry real + feedback* from *vulgar nonsense*, which neither alone can. +4. **Placement** — in `CategoryOverrideResolver`, **after** the rasismi alert override and + an already-demoted model choice, **before** the product category-keyword override: + racism (extreme, single-hit) → dense vulgarity → product hints → model. Ethnic slurs + stay in the alert lexicon (single hit → `rasismi`); this scorer never touches them. +5. **Deterministic, config-driven, no model dependency.** Runs at ingest **and** + restructure, exactly like the other overrides; thresholds are validated config. + +## Consequences + +- `Paskapillupersepornolehtipaviaani` (`paska`+`pillu`+`perse` = 3 distinct, ~0.45 vulgar + share) → **`asiaton`**: unrated, no severity, count-only in "Moderoitava sisältö". + `Möivät paskaa.` (1 distinct, ~0.31 share) → **stays rated** in its real category. +- **Thresholds are empirical and MUST be measured on the owner's real examples**, not + guessed — the defaults here pass the two known cases and are config-tunable; they will + mis-fire on cases outside the fixture until tuned. This ADR does not claim the gate is + precise, only that it is deterministic and honest about its inputs. No invented Finnish + corpus is used for tuning (the eval cases are the owner's). +- **Deferred to a follow-up (Phase 2):** the visible `⚑ kiroilu` **recognition tag** for + rated Level 1–2 items ("flag cursing without demoting"). It needs a structure/schema + field and view work, and is kept out of the operational alert channel (ADR-0033); it is + a separate, single-concern change. The graded *scorer* lands now; only its lower-tier + *presentation* is deferred. +- A domain that ships no `vulgarity-lexicon.json` forces nothing (empty lexicon) — game + and any future domain are unaffected. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index be66059..cf06165 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -49,3 +49,4 @@ these ADRs relocate that reasoning into durable records. | [0036](0036-deterministic-category-keyword-override.md) | Deterministic category-keyword override: a grocery-core lexicon forces product departments (produce → hevi) with cross-category exclusions, sibling to the alert lexicon but raising no alert | Accepted | | [0037](0037-category-keywords-service-premises.md) | Extend the category-keyword override to service/premises (kassa_palvelu, tilat_siisteys) declared last, so products always win and service is the fallback | Accepted | | [0038](0038-unrated-no-trend-no-model-narrative.md) | Unrated content carries no trend either, and demoted categories get a deterministic count-only moderation theme (no model narrative) | Accepted | +| [0039](0039-graded-vulgarity-density-asiaton.md) | Graded vulgarity: a density-gated deterministic scorer forces dense, non-substantive profanity to `asiaton` (a lone swear in real feedback stays rated) | Accepted | diff --git a/domains/retail/vulgarity-lexicon.json b/domains/retail/vulgarity-lexicon.json new file mode 100644 index 0000000..bd2f8c5 --- /dev/null +++ b/domains/retail/vulgarity-lexicon.json @@ -0,0 +1,11 @@ +{ + "matching": "case-insensitive invariant substring over the RAW feedback text; patterns are Finnish profanity STEMS (so inflected forms match). DENSITY-gated (ADR-0039): a message is demoted to 'asiaton' ONLY when the vulgar-character share is >= demoteRatio AND there are >= demoteMinDistinctStems DISTINCT stems — a lone swear in real feedback stays rated, a nonsense pile-up is demoted. Ethnic slurs are NOT here: they live in alert-keywords.json ('rasismi'), which forces the category on a SINGLE hit (ADR-0027). Owner-authored — register is cultural; edit freely.", + "tiers": { + "mild": ["paska", "perse", "hitto", "helvetti", "jumalauta"], + "strong": ["vittu", "kyrpä", "mulkku", "pillu", "runkkari"] + }, + "demoteToCategory": "asiaton", + "demoteRatio": 0.4, + "demoteMinDistinctStems": 2, + "notes": "Thresholds are tuning knobs — measure against real desk examples before trusting them (ADR-0039). demoteRatio is the vulgar-character share of the message; demoteMinDistinctStems counts DISTINCT stems, not occurrences. Defaults chosen so 'Paskapillupersepornolehtipaviaani' (3 distinct, ~0.45 share) demotes while 'Möivät paskaa.' (1 distinct, ~0.31 share) and a furious-but-substantive complaint (low share) stay rated." +} diff --git a/src/FeedbackIntelligence.Api/Ingest/IngestService.cs b/src/FeedbackIntelligence.Api/Ingest/IngestService.cs index 601d8e3..2d0b2d7 100644 --- a/src/FeedbackIntelligence.Api/Ingest/IngestService.cs +++ b/src/FeedbackIntelligence.Api/Ingest/IngestService.cs @@ -21,6 +21,7 @@ public sealed class IngestService( LlmGate llmGate, AlertKeywordSet keywords, CategoryKeywordSet categoryKeywords, + VulgarityLexiconSet vulgarityLexicon, Core.Domain.IActiveDomain activeDomain, Analysis.ReportCache reportCache, ILogger logger) @@ -62,7 +63,7 @@ public async Task IngestAsync(FeedbackRequest request, Cancellat // structure: /interpret already previews the forced category, so a mismatch here // means it was edited away — the rule re-asserts it. var overrideCategory = CategoryOverrideResolver.Resolve( - alerts, request.Text, structure, activeDomain.Descriptor, categoryKeywords.Rules); + alerts, request.Text, structure, activeDomain.Descriptor, categoryKeywords.Rules, vulgarityLexicon.Lexicon); var corrections = request.Corrections; var forcedStructure = AlertMatcher.ApplyCategoryOverride(structure, overrideCategory); if (!ReferenceEquals(forcedStructure, structure)) @@ -173,7 +174,7 @@ public async Task IngestAsync(FeedbackRequest request, Cancellat // store update clears corrections like any restructure, since they // audited a categorization that no longer stands. var forced = AlertMatcher.ApplyCategoryOverride(item.Structure, - CategoryOverrideResolver.Resolve(currentAlerts, item.Text, item.Structure, domain, categoryKeywords.Rules)); + CategoryOverrideResolver.Resolve(currentAlerts, item.Text, item.Structure, domain, categoryKeywords.Rules, vulgarityLexicon.Lexicon)); if (!ReferenceEquals(forced, item.Structure)) { var forcedRows = await store.UpdateStructureAsync( @@ -190,7 +191,7 @@ public async Task IngestAsync(FeedbackRequest request, Cancellat var (structure, itemFailed, notes) = await StructureViaGateAsync(item.Text, $"restructure of {item.Id}", ct); structure = AlertMatcher.ApplyCategoryOverride(structure, - CategoryOverrideResolver.Resolve(currentAlerts, item.Text, structure, domain, categoryKeywords.Rules)); + CategoryOverrideResolver.Resolve(currentAlerts, item.Text, structure, domain, categoryKeywords.Rules, vulgarityLexicon.Lexicon)); var reviewFlags = BuildReviewFlags(item.Text, structure); var updatedRows = await store.UpdateStructureAsync( item.Id, structure, itemFailed, notes, reviewFlags.Count > 0, reviewFlags, ct); diff --git a/src/FeedbackIntelligence.Api/Program.cs b/src/FeedbackIntelligence.Api/Program.cs index 43a1cd0..f4651ed 100644 --- a/src/FeedbackIntelligence.Api/Program.cs +++ b/src/FeedbackIntelligence.Api/Program.cs @@ -38,6 +38,15 @@ var domain = sp.GetRequiredService(); return CategoryKeywordSet.LoadFrom(domain.CategoryKeywordsPath, domain.Descriptor.Categories); }); +// The graded vulgarity lexicon (ADR-0039) is domain data too — optional, and validated +// against the active domain's declared + demoted categories so a typo'd or non-conduct +// demoteToCategory fails the boot rather than re-rating real feedback at the first request. +builder.Services.AddSingleton(sp => +{ + var domain = sp.GetRequiredService(); + return VulgarityLexiconSet.LoadFrom( + domain.VulgarityLexiconPath, domain.Descriptor.Categories, domain.Descriptor.DemotedCategories); +}); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -64,6 +73,7 @@ sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService(), + sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredKeyedService(Channels.Live), sp.GetRequiredService>())); @@ -117,6 +127,7 @@ var reportOptions = app.Services.GetRequiredService>().Value; _ = app.Services.GetRequiredService(); _ = app.Services.GetRequiredService(); // ADR-0036: a typo'd category-keywords.json fails the boot, not the first request +_ = app.Services.GetRequiredService(); // ADR-0039: a bad vulgarity-lexicon.json fails the boot too // The report resolves its prompts from the active domain at generation time. // Validate those roles (and their files) HERE so a domain that omits/misspells a @@ -217,6 +228,7 @@ IOptions options, AlertKeywordSet keywords, CategoryKeywordSet categoryKeywords, + VulgarityLexiconSet vulgarityLexicon, IActiveDomain domain, ILoggerFactory loggerFactory, CancellationToken ct) => @@ -233,7 +245,7 @@ var alerts = AlertMatcher.Match(request.Text, keywords.Categories); var structure = AlertMatcher.ApplyCategoryOverride( result.Structure, - CategoryOverrideResolver.Resolve(alerts, request.Text, result.Structure, domain.Descriptor, categoryKeywords.Rules)); + CategoryOverrideResolver.Resolve(alerts, request.Text, result.Structure, domain.Descriptor, categoryKeywords.Rules, vulgarityLexicon.Lexicon)); return Results.Ok(new { structure, diff --git a/src/FeedbackIntelligence.Api/Structuring/VulgarityLexiconSet.cs b/src/FeedbackIntelligence.Api/Structuring/VulgarityLexiconSet.cs new file mode 100644 index 0000000..666435a --- /dev/null +++ b/src/FeedbackIntelligence.Api/Structuring/VulgarityLexiconSet.cs @@ -0,0 +1,78 @@ +using System.Text.Json; +using FeedbackIntelligence.Core.Structuring; + +namespace FeedbackIntelligence.Api.Structuring; + +/// The active domain's OPTIONAL graded vulgarity lexicon +/// (domains/<active>/vulgarity-lexicon.json, ADR-0039): tiered Finnish profanity stems +/// and the density thresholds that force the demoted conduct category (retail: asiaton). +/// Optional by design — a domain without the file demotes nothing (empty lexicon), so the +/// game domain and any future domain keep working. When the file IS present it is validated +/// at startup: a missing/blank demoteToCategory, one that is not a declared DEMOTED +/// category, or a tiers object with no stems fails the boot rather than silently disabling +/// the layer. +public sealed class VulgarityLexiconSet +{ + public required VulgarityLexicon Lexicon { get; init; } + + public static readonly VulgarityLexiconSet Empty = new() { Lexicon = VulgarityLexicon.Empty }; + + public static VulgarityLexiconSet LoadFrom( + string path, IReadOnlySet declaredCategories, IReadOnlyList demotedCategories) + { + var resolved = FeedbackIntelligence.Llm.AppPathResolver.Resolve(path); + if (!File.Exists(resolved)) + return Empty; // optional layer: no file → demote nothing + + using var doc = JsonDocument.Parse(File.ReadAllText(resolved)); + var root = doc.RootElement; + + if (!root.TryGetProperty("demoteToCategory", out var demoteEl) + || demoteEl.ValueKind != JsonValueKind.String + || string.IsNullOrWhiteSpace(demoteEl.GetString())) + throw new InvalidOperationException($"'{resolved}' must set a non-empty 'demoteToCategory'."); + var demoteTo = demoteEl.GetString()!; + // Forcing an undeclared or non-demoted category would either write an invalid value + // or (worse) re-rate real feedback — fail the boot, like the category-keyword checks. + if (!declaredCategories.Contains(demoteTo)) + throw new InvalidOperationException( + $"'demoteToCategory' \"{demoteTo}\" in '{resolved}' is not a declared category."); + if (!demotedCategories.Contains(demoteTo)) + throw new InvalidOperationException( + $"'demoteToCategory' \"{demoteTo}\" in '{resolved}' is not a DEMOTED category — vulgarity may only " + + "force a conduct/moderation category (ADR-0039)."); + + var mild = ReadTier(root, "mild"); + var strong = ReadTier(root, "strong"); + if (mild.Count == 0 && strong.Count == 0) + throw new InvalidOperationException($"'{resolved}' has no vulgarity stems under 'tiers.mild' / 'tiers.strong'."); + + var ratio = ReadDouble(root, "demoteRatio", VulgarityLexicon.Empty.DemoteRatio); + var minDistinct = ReadInt(root, "demoteMinDistinctStems", VulgarityLexicon.Empty.DemoteMinDistinctStems); + if (ratio is <= 0 or > 1) + throw new InvalidOperationException($"'demoteRatio' in '{resolved}' must be in (0, 1]."); + if (minDistinct < 1) + throw new InvalidOperationException($"'demoteMinDistinctStems' in '{resolved}' must be >= 1."); + + return new VulgarityLexiconSet + { + Lexicon = new VulgarityLexicon(mild, strong, demoteTo, ratio, minDistinct), + }; + } + + private static IReadOnlyList ReadTier(JsonElement root, string tier) => + root.TryGetProperty("tiers", out var tiers) && tiers.ValueKind == JsonValueKind.Object + && tiers.TryGetProperty(tier, out var arr) && arr.ValueKind == JsonValueKind.Array + ? arr.EnumerateArray() + .Where(e => e.ValueKind == JsonValueKind.String) + .Select(e => e.GetString()!) + .Where(s => !string.IsNullOrWhiteSpace(s)) + .ToList() + : []; + + private static double ReadDouble(JsonElement root, string prop, double fallback) => + root.TryGetProperty(prop, out var e) && e.ValueKind == JsonValueKind.Number ? e.GetDouble() : fallback; + + private static int ReadInt(JsonElement root, string prop, int fallback) => + root.TryGetProperty(prop, out var e) && e.ValueKind == JsonValueKind.Number ? e.GetInt32() : fallback; +} diff --git a/src/FeedbackIntelligence.Core/Domain/ActiveDomain.cs b/src/FeedbackIntelligence.Core/Domain/ActiveDomain.cs index 9362f30..2e252a9 100644 --- a/src/FeedbackIntelligence.Core/Domain/ActiveDomain.cs +++ b/src/FeedbackIntelligence.Core/Domain/ActiveDomain.cs @@ -16,6 +16,12 @@ public interface IActiveDomain /// (category-keywords.json, ADR-0036) — term lists that force a category. The file /// need not exist; a domain without it forces nothing. string CategoryKeywordsPath { get; } + + /// Absolute path of the domain's OPTIONAL graded vulgarity lexicon + /// (vulgarity-lexicon.json, ADR-0039) — tiered profanity stems + density thresholds + /// that force the demoted conduct category. The file need not exist; a domain without + /// it demotes nothing. + string VulgarityLexiconPath { get; } string StoriesPath { get; } /// Domain-owned prompt files by role (e.g. "synthesis", @@ -34,6 +40,7 @@ public sealed class ActiveDomain : IActiveDomain public string Name => Descriptor.Name; public string AlertKeywordsPath { get; } public string CategoryKeywordsPath { get; } + public string VulgarityLexiconPath { get; } public string StoriesPath { get; } public IReadOnlyDictionary PromptPaths { get; } @@ -53,6 +60,7 @@ public ActiveDomain(IOptions options) Descriptor = BuildDescriptor(root, o.Active, domainJson); AlertKeywordsPath = Path.Combine(dir, "alert-keywords.json"); CategoryKeywordsPath = Path.Combine(dir, "category-keywords.json"); + VulgarityLexiconPath = Path.Combine(dir, "vulgarity-lexicon.json"); StoriesPath = Path.Combine(dir, "stories.json"); PromptPaths = BuildPromptPaths(root, dir); } diff --git a/src/FeedbackIntelligence.Core/Structuring/CategoryOverrideResolver.cs b/src/FeedbackIntelligence.Core/Structuring/CategoryOverrideResolver.cs index 9ee9b90..0e3970b 100644 --- a/src/FeedbackIntelligence.Core/Structuring/CategoryOverrideResolver.cs +++ b/src/FeedbackIntelligence.Core/Structuring/CategoryOverrideResolver.cs @@ -17,13 +17,20 @@ public static class CategoryOverrideResolver string text, FeedbackStructure? structure, DomainDescriptor descriptor, - IReadOnlyDictionary categoryKeywords) + IReadOnlyDictionary categoryKeywords, + VulgarityLexicon vulgarity) { var alertOverride = AlertMatcher.CategoryOverride(alerts, descriptor.Categories); if (alertOverride is not null) - return alertOverride; + return alertOverride; // racism (single hit → rasismi) outranks everything if (structure is null || descriptor.DemotedCategories.Contains(structure.Category)) - return null; + return null; // nothing to re-categorize, or the model already chose a conduct category + // Dense, non-substantive vulgarity → the demoted conduct category (ADR-0039). + // Ranked BELOW the alert lexicon and an existing demoted choice, ABOVE the product + // category-keyword override — conduct outranks a produce hint. Density-gated, so a + // lone swear inside real feedback falls through here and stays rated. + if (VulgarityScorer.Demotes(text, vulgarity)) + return vulgarity.DemoteToCategory; return CategoryKeywordMatcher.Match(text, categoryKeywords); } } diff --git a/src/FeedbackIntelligence.Core/Structuring/VulgarityScorer.cs b/src/FeedbackIntelligence.Core/Structuring/VulgarityScorer.cs new file mode 100644 index 0000000..20e985b --- /dev/null +++ b/src/FeedbackIntelligence.Core/Structuring/VulgarityScorer.cs @@ -0,0 +1,98 @@ +namespace FeedbackIntelligence.Core.Structuring; + +/// The active domain's OPTIONAL graded vulgarity lexicon (ADR-0039): tiered +/// Finnish profanity stems and the DENSITY thresholds that force the demoted conduct +/// category (retail: asiaton). Density-gated on purpose — a lone swear inside real +/// feedback ("Möivät paskaa") stays rated; a pile-up of distinct stems mashed into +/// nonsense ("Paskapillupersepornolehtipaviaani") is non-substantive and demoted. Ethnic +/// slurs are NOT here — they stay in the alert lexicon (ADR-0027), which forces +/// rasismi on a single hit. when the domain ships no lexicon +/// (forces nothing), so any domain without the file keeps working. +public sealed record VulgarityLexicon( + IReadOnlyList MildStems, + IReadOnlyList StrongStems, + string DemoteToCategory, + // Demote to DemoteToCategory when the vulgar-character SHARE of the message is + // >= DemoteRatio AND at least DemoteMinDistinctStems DISTINCT stems appear. Both + // conditions matter: distinct-count (never raw occurrences) keeps a repeated single + // swear rated; the ratio keeps a furious-but-substantive complaint (mostly real words) + // rated while a nonsense pile-up (mostly vulgar stems) is demoted. + double DemoteRatio, + int DemoteMinDistinctStems) +{ + public static readonly VulgarityLexicon Empty = new([], [], "", 0.4, 2); + + public bool IsEmpty => MildStems.Count == 0 && StrongStems.Count == 0; +} + +/// A graded vulgarity assessment: the (0 none · 1 +/// incidental mild · 2 notable strong · 3 dominant) and whether it s +/// the message to the demoted conduct category. Only Level 3 demotes; Levels 1–2 are +/// recognized-but-rated (a follow-up ⚑ tag reads them, ADR-0039). +public readonly record struct VulgarityAssessment(int Level, bool Demote); + +/// Deterministic, density-gated graded vulgarity scoring (ADR-0039) — a sibling to +/// the alert layer (ADR-0027) and the category-keyword layer (ADR-0036), but it recognizes +/// CONDUCT, not a product department, and demotes only on DENSITY so real feedback that +/// merely swears is never hidden. Runs FIRST and independent of any LLM. +public static class VulgarityScorer +{ + /// Assess text against the lexicon. Distinct stems are counted once each + /// (repeating one swear does not inflate the score); the ratio is total matched + /// characters over the message's non-whitespace length. + public static VulgarityAssessment Assess(string text, VulgarityLexicon lex) + { + if (lex.IsEmpty || string.IsNullOrWhiteSpace(text)) + return new VulgarityAssessment(0, false); + + var distinct = 0; + var matchedChars = 0; + var anyStrong = false; + + foreach (var stem in lex.MildStems) + { + var n = CountOccurrences(text, stem); + if (n == 0) continue; + distinct++; + matchedChars += n * stem.Length; + } + foreach (var stem in lex.StrongStems) + { + var n = CountOccurrences(text, stem); + if (n == 0) continue; + distinct++; + matchedChars += n * stem.Length; + anyStrong = true; + } + + if (distinct == 0) + return new VulgarityAssessment(0, false); + + var nonWs = text.Count(c => !char.IsWhiteSpace(c)); + var ratio = nonWs > 0 ? (double)matchedChars / nonWs : 0.0; + var demote = ratio >= lex.DemoteRatio && distinct >= lex.DemoteMinDistinctStems; + var level = demote ? 3 : (anyStrong ? 2 : 1); + return new VulgarityAssessment(level, demote); + } + + /// Does the text's vulgarity DENSITY force the demoted conduct category? + /// The single seam asks through. + public static bool Demotes(string text, VulgarityLexicon lex) => + !string.IsNullOrEmpty(lex.DemoteToCategory) && Assess(text, lex).Demote; + + /// Non-overlapping, case-insensitive invariant occurrences of a stem — the + /// same substring contract as the alert / category-keyword lexicons. + private static int CountOccurrences(string text, string stem) + { + if (string.IsNullOrEmpty(stem)) + return 0; + var count = 0; + var idx = 0; + while ((idx = text.IndexOf(stem, idx, StringComparison.InvariantCultureIgnoreCase)) >= 0) + { + count++; + idx += stem.Length; + } + return count; + } +} diff --git a/tests/FeedbackIntelligence.Api.Tests/CategoryOverrideResolverTests.cs b/tests/FeedbackIntelligence.Api.Tests/CategoryOverrideResolverTests.cs index 99912d2..f890a42 100644 --- a/tests/FeedbackIntelligence.Api.Tests/CategoryOverrideResolverTests.cs +++ b/tests/FeedbackIntelligence.Api.Tests/CategoryOverrideResolverTests.cs @@ -3,14 +3,16 @@ namespace FeedbackIntelligence.Api.Tests; -/// ADR-0036: the ONE place that decides which category is forced — -/// the alert lexicon (ADR-0027) always outranks the category-keyword lexicon, -/// and neither ever overrides a demoted (conduct) category already assigned. +/// ADR-0036/0039: the ONE place that decides which category is forced — +/// the alert lexicon (ADR-0027, racism) always outranks dense vulgarity (ADR-0039), +/// which outranks the category-keyword lexicon (ADR-0036), and none ever overrides a +/// demoted (conduct) category already assigned. public class CategoryOverrideResolverTests { private static readonly Core.Domain.DomainDescriptor Retail = TestDomains.Retail(); private static readonly IReadOnlyDictionary CategoryKeywords = TestDomains.RetailCategoryKeywords().Rules; + private static readonly VulgarityLexicon Vulgarity = TestDomains.RetailVulgarity().Lexicon; [Fact] public void AlertHit_WinsOverProduceTerm() @@ -18,7 +20,7 @@ public void AlertHit_WinsOverProduceTerm() var alerts = new List { new("rasismi", "neeker") }; var structure = new FeedbackStructure("maito_kylma", "teema", "high", "complaint", "fi"); - var result = CategoryOverrideResolver.Resolve(alerts, "nektariini", structure, Retail, CategoryKeywords); + var result = CategoryOverrideResolver.Resolve(alerts, "nektariini", structure, Retail, CategoryKeywords, Vulgarity); Assert.Equal("rasismi", result); } @@ -28,7 +30,7 @@ public void NoAlert_ProduceTerm_ForcesCategoryKeyword() { var structure = new FeedbackStructure("maito_kylma", "teema", "high", "complaint", "fi"); - var result = CategoryOverrideResolver.Resolve([], "nektariinierä", structure, Retail, CategoryKeywords); + var result = CategoryOverrideResolver.Resolve([], "nektariinierä", structure, Retail, CategoryKeywords, Vulgarity); Assert.Equal("hevi", result); } @@ -38,7 +40,7 @@ public void NoAlert_ExistingCategoryIsDemoted_NeverOverridden() { var structure = new FeedbackStructure("rasismi", "teema", "high", "complaint", "fi"); - var result = CategoryOverrideResolver.Resolve([], "nektariini", structure, Retail, CategoryKeywords); + var result = CategoryOverrideResolver.Resolve([], "nektariini", structure, Retail, CategoryKeywords, Vulgarity); Assert.Null(result); } @@ -50,7 +52,7 @@ public void NoAlert_NoTerm_ReturnsNull() // ADR-0037: 'palvelu' is now itself a kassa_palvelu term, so this needs text with // no category-keyword term at all (product OR service/premises) to stay null. - var result = CategoryOverrideResolver.Resolve([], "ihan tavallinen kokemus", structure, Retail, CategoryKeywords); + var result = CategoryOverrideResolver.Resolve([], "ihan tavallinen kokemus", structure, Retail, CategoryKeywords, Vulgarity); Assert.Null(result); } @@ -58,8 +60,48 @@ public void NoAlert_NoTerm_ReturnsNull() [Fact] public void NoAlert_NullStructure_ReturnsNull() { - var result = CategoryOverrideResolver.Resolve([], "nektariini", null, Retail, CategoryKeywords); + var result = CategoryOverrideResolver.Resolve([], "nektariini", null, Retail, CategoryKeywords, Vulgarity); Assert.Null(result); } + + [Fact] + public void DenseVulgarity_ForcesAsiaton_OverModelCategory() + { + // ADR-0039: a pile-up of distinct vulgar stems mashed into nonsense — the model + // filed it under the rated catch-all 'muu'; the density override moves it to asiaton. + var structure = new FeedbackStructure("muu", "offensive_language", "high", "other", "fi"); + + var result = CategoryOverrideResolver.Resolve( + [], "Paskapillupersepornolehtipaviaani", structure, Retail, CategoryKeywords, Vulgarity); + + Assert.Equal("asiaton", result); + } + + [Fact] + public void LoneSwearInRealComplaint_StaysRated_NotDemoted() + { + // ADR-0039: one stem inside substantive feedback is a crude-but-real complaint — + // density is too low to demote, so it keeps its rated category (no override here). + var structure = new FeedbackStructure("muu", "laatu", "high", "complaint", "fi"); + + var result = CategoryOverrideResolver.Resolve( + [], "Möivät paskaa. Epäilyttävää paskaa.", structure, Retail, CategoryKeywords, Vulgarity); + + Assert.Null(result); // no override → the model's rated category stands + } + + [Fact] + public void RacismAlert_WinsOverDenseVulgarity() + { + // Racism (single-hit alert, ADR-0027) outranks the vulgarity density override: + // a message that is BOTH a slur-alert and dense vulgarity is filed rasismi, not asiaton. + var alerts = new List { new("rasismi", "neeker") }; + var structure = new FeedbackStructure("muu", "teema", "high", "other", "fi"); + + var result = CategoryOverrideResolver.Resolve( + alerts, "neekeri paska vittu perse", structure, Retail, CategoryKeywords, Vulgarity); + + Assert.Equal("rasismi", result); + } } diff --git a/tests/FeedbackIntelligence.Api.Tests/ChannelIsolationTests.cs b/tests/FeedbackIntelligence.Api.Tests/ChannelIsolationTests.cs index e75ec05..825b03c 100644 --- a/tests/FeedbackIntelligence.Api.Tests/ChannelIsolationTests.cs +++ b/tests/FeedbackIntelligence.Api.Tests/ChannelIsolationTests.cs @@ -51,6 +51,7 @@ public void Dispose() Categories = new Dictionary> { ["injury_safety"] = ["loukkaantu"] }, }, Api.Structuring.CategoryKeywordSet.Empty, + Api.Structuring.VulgarityLexiconSet.Empty, TestDomains.RetailActive(), cache, NullLogger.Instance); diff --git a/tests/FeedbackIntelligence.Api.Tests/IngestServiceTests.cs b/tests/FeedbackIntelligence.Api.Tests/IngestServiceTests.cs index 947c006..7374427 100644 --- a/tests/FeedbackIntelligence.Api.Tests/IngestServiceTests.cs +++ b/tests/FeedbackIntelligence.Api.Tests/IngestServiceTests.cs @@ -41,6 +41,7 @@ public void Dispose() Categories = new Dictionary> { ["injury_safety"] = ["loukkaantu"] }, }, Structuring.CategoryKeywordSet.Empty, + Structuring.VulgarityLexiconSet.Empty, TestDomains.RetailActive(), new Analysis.ReportCache(), NullLogger.Instance); @@ -54,6 +55,7 @@ public void Dispose() new LlmGate(_options), TestDomains.RetailKeywords(), TestDomains.RetailCategoryKeywords(), + TestDomains.RetailVulgarity(), TestDomains.RetailActive(), new Analysis.ReportCache(), NullLogger.Instance); @@ -333,6 +335,7 @@ await _store.InsertAsync(new StoredFeedback( Categories = new Dictionary> { ["injury_safety"] = ["loukkaantu"] }, }, Structuring.CategoryKeywordSet.Empty, + Structuring.VulgarityLexiconSet.Empty, TestDomains.RetailActive(), cache, NullLogger.Instance); @@ -455,6 +458,7 @@ public async Task CategoryKeyword_ForcesCategory_OverModelStructure() Categories = new Dictionary> { ["injury_safety"] = ["loukkaantu"] }, }, Structuring.CategoryKeywordSet.Empty, + Structuring.VulgarityLexiconSet.Empty, TestDomains.RetailActive(), new Analysis.ReportCache(), NullLogger.Instance); diff --git a/tests/FeedbackIntelligence.Api.Tests/TestDomains.cs b/tests/FeedbackIntelligence.Api.Tests/TestDomains.cs index bbe5684..cf54d64 100644 --- a/tests/FeedbackIntelligence.Api.Tests/TestDomains.cs +++ b/tests/FeedbackIntelligence.Api.Tests/TestDomains.cs @@ -31,6 +31,13 @@ public static Api.Structuring.CategoryKeywordSet RetailCategoryKeywords() => Path.Combine(RepoRoot(), "domains", "retail", "category-keywords.json"), Retail().Categories); + /// The COMMITTED retail vulgarity lexicon (ADR-0039) — the one place tests + /// resolve its path, so a module move breaks exactly one helper. + public static Api.Structuring.VulgarityLexiconSet RetailVulgarity() => + Api.Structuring.VulgarityLexiconSet.LoadFrom( + Path.Combine(RepoRoot(), "domains", "retail", "vulgarity-lexicon.json"), + Retail().Categories, Retail().DemotedCategories); + internal static string RepoRoot() { var dir = new DirectoryInfo(AppContext.BaseDirectory); @@ -47,6 +54,7 @@ public StubActiveDomain(DomainDescriptor descriptor, string? promptOverride) var dir = Path.Combine(RepoRoot(), "domains", "retail"); AlertKeywordsPath = Path.Combine(dir, "alert-keywords.json"); CategoryKeywordsPath = Path.Combine(dir, "category-keywords.json"); + VulgarityLexiconPath = Path.Combine(dir, "vulgarity-lexicon.json"); StoriesPath = Path.Combine(dir, "stories.json"); PromptPaths = new Dictionary(StringComparer.Ordinal) { @@ -60,6 +68,7 @@ public StubActiveDomain(DomainDescriptor descriptor, string? promptOverride) public string Name => Descriptor.Name; public string AlertKeywordsPath { get; } public string CategoryKeywordsPath { get; } + public string VulgarityLexiconPath { get; } public string StoriesPath { get; } public IReadOnlyDictionary PromptPaths { get; } public string PromptPath(string role) => PromptPaths.TryGetValue(role, out var p) diff --git a/tests/FeedbackIntelligence.Api.Tests/VulgarityLexiconSetTests.cs b/tests/FeedbackIntelligence.Api.Tests/VulgarityLexiconSetTests.cs new file mode 100644 index 0000000..7778854 --- /dev/null +++ b/tests/FeedbackIntelligence.Api.Tests/VulgarityLexiconSetTests.cs @@ -0,0 +1,58 @@ +using FeedbackIntelligence.Api.Structuring; + +namespace FeedbackIntelligence.Api.Tests; + +/// ADR-0039: the vulgarity lexicon is optional domain data, validated at startup — +/// a demoteToCategory that is not a declared, DEMOTED category fails the boot rather than +/// silently re-rating real feedback; a missing file demotes nothing. +public class VulgarityLexiconSetTests +{ + private static readonly IReadOnlySet Declared = TestDomains.Retail().Categories; + private static readonly IReadOnlyList Demoted = TestDomains.Retail().DemotedCategories; + + [Fact] + public void CommittedRetailLexicon_Loads_DemotesToAsiaton() + { + var set = TestDomains.RetailVulgarity(); + Assert.Equal("asiaton", set.Lexicon.DemoteToCategory); + Assert.False(set.Lexicon.IsEmpty); + Assert.Contains("paska", set.Lexicon.MildStems); + Assert.Contains("vittu", set.Lexicon.StrongStems); + } + + [Fact] + public void MissingFile_ReturnsEmpty_DemotesNothing() + { + var set = VulgarityLexiconSet.LoadFrom( + Path.Combine(Path.GetTempPath(), $"no-such-{Guid.NewGuid():N}.json"), Declared, Demoted); + Assert.True(set.Lexicon.IsEmpty); + } + + [Fact] + public void DemoteToCategory_NotADemotedCategory_FailsBoot() + { + // 'muu' is a declared category but NOT demoted — forcing it would re-rate real feedback. + var path = Path.Combine(Path.GetTempPath(), $"vulg-{Guid.NewGuid():N}.json"); + File.WriteAllText(path, """{"tiers":{"mild":["paska"]},"demoteToCategory":"muu"}"""); + try + { + var ex = Assert.Throws( + () => VulgarityLexiconSet.LoadFrom(path, Declared, Demoted)); + Assert.Contains("not a DEMOTED category", ex.Message); + } + finally { File.Delete(path); } + } + + [Fact] + public void DemoteToCategory_Undeclared_FailsBoot() + { + var path = Path.Combine(Path.GetTempPath(), $"vulg-{Guid.NewGuid():N}.json"); + File.WriteAllText(path, """{"tiers":{"mild":["paska"]},"demoteToCategory":"ei_olemassa"}"""); + try + { + Assert.Throws( + () => VulgarityLexiconSet.LoadFrom(path, Declared, Demoted)); + } + finally { File.Delete(path); } + } +} diff --git a/tests/FeedbackIntelligence.Api.Tests/VulgarityScorerTests.cs b/tests/FeedbackIntelligence.Api.Tests/VulgarityScorerTests.cs new file mode 100644 index 0000000..e2ff8c3 --- /dev/null +++ b/tests/FeedbackIntelligence.Api.Tests/VulgarityScorerTests.cs @@ -0,0 +1,66 @@ +using FeedbackIntelligence.Core.Structuring; + +namespace FeedbackIntelligence.Api.Tests; + +/// ADR-0039: density-gated graded vulgarity. Demote (Level 3 → asiaton) only when +/// the vulgar-character share is high AND enough DISTINCT stems appear — so a lone swear in +/// real feedback, or a repeated single swear, stays rated, while a nonsense pile-up is +/// demoted. Uses the COMMITTED retail lexicon so the tuned thresholds are what ships. +public class VulgarityScorerTests +{ + private static readonly VulgarityLexicon Lex = TestDomains.RetailVulgarity().Lexicon; + + [Fact] + public void DensePileUp_OfDistinctStems_Demotes() + { + // paska + pillu + perse = 3 distinct, ~0.45 vulgar share → Level 3 → asiaton. + var a = VulgarityScorer.Assess("Paskapillupersepornolehtipaviaani", Lex); + Assert.True(a.Demote); + Assert.Equal(3, a.Level); + Assert.True(VulgarityScorer.Demotes("Paskapillupersepornolehtipaviaani", Lex)); + } + + [Fact] + public void LoneSwearInRealComplaint_DoesNotDemote() + { + // one distinct stem, low share → a crude-but-real complaint stays rated. + Assert.False(VulgarityScorer.Assess("Möivät paskaa. Epäilyttävää paskaa.", Lex).Demote); + } + + [Fact] + public void RepeatedSingleStem_StaysRated_DistinctGate() + { + // "singletons shouldn't set a big flag" — even repeated, ONE distinct stem never demotes. + Assert.False(VulgarityScorer.Demotes("paska paska paska paska", Lex)); + } + + [Fact] + public void FuriousButSubstantiveComplaint_WithThreeSwears_StaysRated() + { + // Three DISTINCT stems (paska/kyrpä/perse) but embedded in real words → low vulgar + // share → rated. This is the case raw distinct-count alone would wrongly demote. + const string text = "Tämä tuote on ihan paska ja teidän palvelunne on kyrpää, en osta enää mitään perseestä."; + Assert.False(VulgarityScorer.Assess(text, Lex).Demote); + } + + [Fact] + public void ContentFreeTwoStemWall_Demotes() + { + // no substance, all vulgar → demote even at two distinct stems. + Assert.True(VulgarityScorer.Demotes("vittu perse", Lex)); + } + + [Fact] + public void CleanText_IsLevelZero_NoDemote() + { + var a = VulgarityScorer.Assess("Kurpitsojen laatu vaikuttaa olevan parempi nyt", Lex); + Assert.Equal(0, a.Level); + Assert.False(a.Demote); + } + + [Fact] + public void EmptyLexicon_NeverDemotes() + { + Assert.False(VulgarityScorer.Demotes("paska perse vittu mulkku", VulgarityLexicon.Empty)); + } +} diff --git a/tests/FeedbackIntelligence.Llm.Tests/LlmStructuringServiceTests.cs b/tests/FeedbackIntelligence.Llm.Tests/LlmStructuringServiceTests.cs index 8f69ce9..4b59114 100644 --- a/tests/FeedbackIntelligence.Llm.Tests/LlmStructuringServiceTests.cs +++ b/tests/FeedbackIntelligence.Llm.Tests/LlmStructuringServiceTests.cs @@ -132,6 +132,7 @@ private sealed class StubDomain(DomainDescriptor descriptor) : IActiveDomain public string Name => Descriptor.Name; public string AlertKeywordsPath => ""; public string CategoryKeywordsPath => ""; + public string VulgarityLexiconPath => ""; public string StoriesPath => ""; public IReadOnlyDictionary PromptPaths { get; } = new Dictionary(StringComparer.Ordinal); From f7ea960a479bff64804ac3abada5d5c30d7b4b0c Mon Sep 17 00:00:00 2001 From: Mikko Numminen Date: Wed, 15 Jul 2026 05:33:06 +0300 Subject: [PATCH 2/2] fix(retail): harden vulgarity scorer per review (NFD crash, overlap/dedup, threshold validation) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review of the diff surfaced three owner-config-path issues (none fire with the committed lexicon, but the lexicon is owner-authored, so they would): - CONFIRMED crash: the match loop advanced the cursor by the STEM length under culture-aware (InvariantCulture) matching, which can match a span SHORTER than the stem (an NFD-authored Finnish stem — ä/ö have NFC/NFD ambiguity — against NFC text), overshooting the end and throwing ArgumentOutOfRangeException on /interpret + ingest. Now advances by the ACTUAL matched length via the span IndexOf(out matchLength) overload; can never over-/under-run the buffer. - Overlap/dedup: matchedChars summed n*stem.Length, so overlapping/substring stems double-counted and the ratio could exceed 1; and a stem listed in both tiers inflated the distinct-count, defeating the gate that keeps a repeated single swear rated. Now a coverage MASK bounds the ratio at <=1 and stems are de-duplicated across tiers (counted once). - Loader: demoteRatio / demoteMinDistinctStems silently fell back to defaults on a present-but-wrong-typed value (a quoted "0.6" or null), unlike the strict demoteToCategory check. Now a present non-number fails the boot. Also corrects an ADR figure (~0.31 -> ~0.38 share for "Möivät paskaa.", noting the distinct gate — not the ratio — is what keeps it rated). Adds regression tests: NFD-authored stem matches NFC text without crashing; duplicate stem across tiers stays rated; wrong-typed threshold fails the boot. Full solution green (Api 166). --- .../0039-graded-vulgarity-density-asiaton.md | 3 +- domains/retail/vulgarity-lexicon.json | 2 +- .../Structuring/VulgarityLexiconSet.cs | 27 ++++-- .../Structuring/VulgarityScorer.cs | 83 ++++++++++++------- .../VulgarityLexiconSetTests.cs | 16 ++++ .../VulgarityScorerTests.cs | 25 ++++++ 6 files changed, 118 insertions(+), 38 deletions(-) diff --git a/docs/decisions/0039-graded-vulgarity-density-asiaton.md b/docs/decisions/0039-graded-vulgarity-density-asiaton.md index bd6838a..a63150b 100644 --- a/docs/decisions/0039-graded-vulgarity-density-asiaton.md +++ b/docs/decisions/0039-graded-vulgarity-density-asiaton.md @@ -59,7 +59,8 @@ category (`asiaton`).** - `Paskapillupersepornolehtipaviaani` (`paska`+`pillu`+`perse` = 3 distinct, ~0.45 vulgar share) → **`asiaton`**: unrated, no severity, count-only in "Moderoitava sisältö". - `Möivät paskaa.` (1 distinct, ~0.31 share) → **stays rated** in its real category. + `Möivät paskaa.` (1 distinct, ~0.38 share — the *distinct* gate, not the ratio, is what + keeps it rated) → **stays rated** in its real category. - **Thresholds are empirical and MUST be measured on the owner's real examples**, not guessed — the defaults here pass the two known cases and are config-tunable; they will mis-fire on cases outside the fixture until tuned. This ADR does not claim the gate is diff --git a/domains/retail/vulgarity-lexicon.json b/domains/retail/vulgarity-lexicon.json index bd2f8c5..f548882 100644 --- a/domains/retail/vulgarity-lexicon.json +++ b/domains/retail/vulgarity-lexicon.json @@ -7,5 +7,5 @@ "demoteToCategory": "asiaton", "demoteRatio": 0.4, "demoteMinDistinctStems": 2, - "notes": "Thresholds are tuning knobs — measure against real desk examples before trusting them (ADR-0039). demoteRatio is the vulgar-character share of the message; demoteMinDistinctStems counts DISTINCT stems, not occurrences. Defaults chosen so 'Paskapillupersepornolehtipaviaani' (3 distinct, ~0.45 share) demotes while 'Möivät paskaa.' (1 distinct, ~0.31 share) and a furious-but-substantive complaint (low share) stay rated." + "notes": "Thresholds are tuning knobs — measure against real desk examples before trusting them (ADR-0039). demoteRatio is the vulgar-character share of the message; demoteMinDistinctStems counts DISTINCT stems, not occurrences. Defaults chosen so 'Paskapillupersepornolehtipaviaani' (3 distinct, ~0.45 share) demotes while 'Möivät paskaa.' (1 distinct; the distinct gate keeps it rated) and a furious-but-substantive complaint (low share) stay rated." } diff --git a/src/FeedbackIntelligence.Api/Structuring/VulgarityLexiconSet.cs b/src/FeedbackIntelligence.Api/Structuring/VulgarityLexiconSet.cs index 666435a..6251e77 100644 --- a/src/FeedbackIntelligence.Api/Structuring/VulgarityLexiconSet.cs +++ b/src/FeedbackIntelligence.Api/Structuring/VulgarityLexiconSet.cs @@ -47,8 +47,8 @@ public static VulgarityLexiconSet LoadFrom( if (mild.Count == 0 && strong.Count == 0) throw new InvalidOperationException($"'{resolved}' has no vulgarity stems under 'tiers.mild' / 'tiers.strong'."); - var ratio = ReadDouble(root, "demoteRatio", VulgarityLexicon.Empty.DemoteRatio); - var minDistinct = ReadInt(root, "demoteMinDistinctStems", VulgarityLexicon.Empty.DemoteMinDistinctStems); + var ratio = ReadDouble(root, "demoteRatio", VulgarityLexicon.Empty.DemoteRatio, resolved); + var minDistinct = ReadInt(root, "demoteMinDistinctStems", VulgarityLexicon.Empty.DemoteMinDistinctStems, resolved); if (ratio is <= 0 or > 1) throw new InvalidOperationException($"'demoteRatio' in '{resolved}' must be in (0, 1]."); if (minDistinct < 1) @@ -70,9 +70,24 @@ private static IReadOnlyList ReadTier(JsonElement root, string tier) => .ToList() : []; - private static double ReadDouble(JsonElement root, string prop, double fallback) => - root.TryGetProperty(prop, out var e) && e.ValueKind == JsonValueKind.Number ? e.GetDouble() : fallback; + // Absent → the documented default; present but not a number → fail the boot (a quoted + // "0.6" or a null was meant to tune the gate and must not be silently ignored, matching + // the strict demoteToCategory validation above). + private static double ReadDouble(JsonElement root, string prop, double fallback, string resolved) + { + if (!root.TryGetProperty(prop, out var e)) + return fallback; + if (e.ValueKind != JsonValueKind.Number) + throw new InvalidOperationException($"'{prop}' in '{resolved}' must be a number."); + return e.GetDouble(); + } - private static int ReadInt(JsonElement root, string prop, int fallback) => - root.TryGetProperty(prop, out var e) && e.ValueKind == JsonValueKind.Number ? e.GetInt32() : fallback; + private static int ReadInt(JsonElement root, string prop, int fallback, string resolved) + { + if (!root.TryGetProperty(prop, out var e)) + return fallback; + if (e.ValueKind != JsonValueKind.Number) + throw new InvalidOperationException($"'{prop}' in '{resolved}' must be a number."); + return e.GetInt32(); + } } diff --git a/src/FeedbackIntelligence.Core/Structuring/VulgarityScorer.cs b/src/FeedbackIntelligence.Core/Structuring/VulgarityScorer.cs index 20e985b..7f286cf 100644 --- a/src/FeedbackIntelligence.Core/Structuring/VulgarityScorer.cs +++ b/src/FeedbackIntelligence.Core/Structuring/VulgarityScorer.cs @@ -1,3 +1,5 @@ +using System.Globalization; + namespace FeedbackIntelligence.Core.Structuring; /// The active domain's OPTIONAL graded vulgarity lexicon (ADR-0039): tiered @@ -37,38 +39,36 @@ public sealed record VulgarityLexicon( /// merely swears is never hidden. Runs FIRST and independent of any LLM. public static class VulgarityScorer { - /// Assess text against the lexicon. Distinct stems are counted once each - /// (repeating one swear does not inflate the score); the ratio is total matched - /// characters over the message's non-whitespace length. + private static readonly CompareInfo Invariant = CultureInfo.InvariantCulture.CompareInfo; + + /// Assess text against the lexicon. A stem is counted DISTINCT once even if it + /// appears in both tiers (dedup), so a repeated single swear can never clear the + /// distinct gate. The vulgar-character share uses a COVERAGE MASK, so overlapping or + /// substring stems can never double-count and the ratio can never exceed 1. public static VulgarityAssessment Assess(string text, VulgarityLexicon lex) { if (lex.IsEmpty || string.IsNullOrWhiteSpace(text)) return new VulgarityAssessment(0, false); + var covered = new bool[text.Length]; + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); var distinct = 0; - var matchedChars = 0; var anyStrong = false; - foreach (var stem in lex.MildStems) - { - var n = CountOccurrences(text, stem); - if (n == 0) continue; - distinct++; - matchedChars += n * stem.Length; - } - foreach (var stem in lex.StrongStems) - { - var n = CountOccurrences(text, stem); - if (n == 0) continue; - distinct++; - matchedChars += n * stem.Length; - anyStrong = true; - } + foreach (var (stem, strong) in DistinctStems(lex, seen)) + if (MarkOccurrences(text, stem, covered)) + { + distinct++; + if (strong) anyStrong = true; + } if (distinct == 0) return new VulgarityAssessment(0, false); - var nonWs = text.Count(c => !char.IsWhiteSpace(c)); + var matchedChars = 0; + foreach (var c in covered) + if (c) matchedChars++; + var nonWs = text.Count(ch => !char.IsWhiteSpace(ch)); var ratio = nonWs > 0 ? (double)matchedChars / nonWs : 0.0; var demote = ratio >= lex.DemoteRatio && distinct >= lex.DemoteMinDistinctStems; var level = demote ? 3 : (anyStrong ? 2 : 1); @@ -80,19 +80,42 @@ public static VulgarityAssessment Assess(string text, VulgarityLexicon lex) public static bool Demotes(string text, VulgarityLexicon lex) => !string.IsNullOrEmpty(lex.DemoteToCategory) && Assess(text, lex).Demote; - /// Non-overlapping, case-insensitive invariant occurrences of a stem — the - /// same substring contract as the alert / category-keyword lexicons. - private static int CountOccurrences(string text, string stem) + // Mild first, then strong, each stem yielded ONCE across both tiers (a stem listed in + // both is not double-counted — that would defeat the distinct gate a repeated single + // swear relies on to stay rated). + private static IEnumerable<(string Stem, bool Strong)> DistinctStems(VulgarityLexicon lex, HashSet seen) + { + foreach (var s in lex.MildStems) + if (!string.IsNullOrEmpty(s) && seen.Add(s)) + yield return (s, false); + foreach (var s in lex.StrongStems) + if (!string.IsNullOrEmpty(s) && seen.Add(s)) + yield return (s, true); + } + + /// Mark every case-insensitive invariant occurrence of a stem on the coverage + /// mask; returns whether the stem matched at least once. Advances by the ACTUAL matched + /// length (not the stem's length) — culture-aware matching can match a span shorter than + /// the stem (e.g. an NFD-authored stem against NFC text), so advancing by the stem length + /// could overshoot the end of the string and throw. Same substring contract as the alert + /// / category-keyword lexicons; can never over- or under-run the buffer. + private static bool MarkOccurrences(string text, string stem, bool[] covered) { - if (string.IsNullOrEmpty(stem)) - return 0; - var count = 0; + var any = false; + var stemSpan = stem.AsSpan(); var idx = 0; - while ((idx = text.IndexOf(stem, idx, StringComparison.InvariantCultureIgnoreCase)) >= 0) + while (idx < text.Length) { - count++; - idx += stem.Length; + var rel = Invariant.IndexOf(text.AsSpan(idx), stemSpan, CompareOptions.IgnoreCase, out var matchLen); + if (rel < 0) + break; + any = true; + var found = idx + rel; + var end = Math.Min(found + matchLen, text.Length); + for (var i = found; i < end; i++) + covered[i] = true; + idx = found + Math.Max(matchLen, 1); } - return count; + return any; } } diff --git a/tests/FeedbackIntelligence.Api.Tests/VulgarityLexiconSetTests.cs b/tests/FeedbackIntelligence.Api.Tests/VulgarityLexiconSetTests.cs index 7778854..41d2b67 100644 --- a/tests/FeedbackIntelligence.Api.Tests/VulgarityLexiconSetTests.cs +++ b/tests/FeedbackIntelligence.Api.Tests/VulgarityLexiconSetTests.cs @@ -55,4 +55,20 @@ public void DemoteToCategory_Undeclared_FailsBoot() } finally { File.Delete(path); } } + + [Fact] + public void Threshold_PresentButNotANumber_FailsBoot() + { + // A quoted "0.6" was meant to tune the gate — silently falling back to the default + // would run the gate looser than configured, so it must fail the boot (not fall back). + var path = Path.Combine(Path.GetTempPath(), $"vulg-{Guid.NewGuid():N}.json"); + File.WriteAllText(path, """{"tiers":{"mild":["paska"]},"demoteToCategory":"asiaton","demoteRatio":"0.6"}"""); + try + { + var ex = Assert.Throws( + () => VulgarityLexiconSet.LoadFrom(path, Declared, Demoted)); + Assert.Contains("must be a number", ex.Message); + } + finally { File.Delete(path); } + } } diff --git a/tests/FeedbackIntelligence.Api.Tests/VulgarityScorerTests.cs b/tests/FeedbackIntelligence.Api.Tests/VulgarityScorerTests.cs index e2ff8c3..6aee208 100644 --- a/tests/FeedbackIntelligence.Api.Tests/VulgarityScorerTests.cs +++ b/tests/FeedbackIntelligence.Api.Tests/VulgarityScorerTests.cs @@ -63,4 +63,29 @@ public void EmptyLexicon_NeverDemotes() { Assert.False(VulgarityScorer.Demotes("paska perse vittu mulkku", VulgarityLexicon.Empty)); } + + [Fact] + public void NfdAuthoredStem_MatchesNfcText_WithoutOverrunningTheBuffer() + { + // A stem authored in DECOMPOSED form: 'kyrp' + 'a' + U+0308 combining diaeresis = 6 code + // units, canonically equal to the composed 'kyrp' + U+00E4. Matched against an NFC + // message ending in the composed form (matched span = 5 units), culture-aware matching + // spans FEWER units than the stem, so advancing by the STEM length would index past the + // end and throw. Must still match (canonical equivalence) and must not crash. Built from + // (char) casts so the normalization form is exact regardless of source-file encoding. + var nfdStem = "kyrp" + 'a' + (char)0x0308; // decomposed + var nfcText = "paska kyrp" + (char)0x00e4; // composed, match at the end of the string + var lex = new VulgarityLexicon(["paska"], [nfdStem], "asiaton", 0.4, 2); + Assert.True(VulgarityScorer.Demotes(nfcText, lex)); + } + + [Fact] + public void StemListedInBothTiers_IsCountedOnce_RepeatedSwearStaysRated() + { + // A copy-paste duplicate across tiers must NOT inflate the distinct-count — otherwise a + // repeated single swear ("paska paska") would clear the distinct gate and be demoted, + // the exact case ADR-0039 says stays rated. + var lex = new VulgarityLexicon(["paska"], ["paska"], "asiaton", 0.4, 2); + Assert.False(VulgarityScorer.Demotes("paska paska", lex)); + } }