From 11d249fe58dd2497c6b6e4a05f4e7afb763fae99 Mon Sep 17 00:00:00 2001 From: Mikko Numminen Date: Wed, 15 Jul 2026 03:18:43 +0300 Subject: [PATCH] fix(report): live-summary Yhteenveto excludes demoted content (ADR-0032/0033) The whole-window Overall (Yhteenveto) was synthesized over every structured item, including demoted rasismi/asiaton ones. SynthesizeThemeAsync feeds the model a severity distribution and per-item excerpts with severity, so the lead summary named moderated content with a rating ("'' (korkea)") and counted its severities in the distribution; the Overall count and trend (the desk "suunta" tile) were over all items too. Synthesize the Overall over rated items only (excluding DemotedCategories): narrative, severity digest, excerpts, window total, trend, and needs-review count. reportSentiment already excluded demoted. A window of only demoted content now yields no Yhteenveto (the moderation count still shows), instead of a summary written about hostile content. This brings the lead summary in line with ADR-0033 section 3 (top of page = real feedback only) and ADR-0032 (demoted = unrated, excluded from severity aggregates). Adds a regression test asserting the demoted item's id/excerpt/critical severity never reach the synthesis digest and Overall.Count/FeedbackIds are rated-only. --- ...0033-operational-alerts-moderation-view.md | 8 ++- .../Analysis/ReportService.cs | 50 ++++++++++------ .../ReportServiceTests.cs | 58 +++++++++++++++++++ 3 files changed, 98 insertions(+), 18 deletions(-) diff --git a/docs/decisions/0033-operational-alerts-moderation-view.md b/docs/decisions/0033-operational-alerts-moderation-view.md index 8c6c062..87a3b98 100644 --- a/docs/decisions/0033-operational-alerts-moderation-view.md +++ b/docs/decisions/0033-operational-alerts-moderation-view.md @@ -44,7 +44,13 @@ it should be recognized but never lead the page. "Palautteet aiheittain" category chart and the entries tile — the top of the page reflects **real feedback only**; the moderation disclosure carries the conduct count. (Severity and sentiment already excluded the unrated categories - per ADR-0032.) + per ADR-0032.) This includes the whole-window **Yhteenveto** (the live-summary + `Overall`, ADR-0026): it is synthesized over the **rated** items only, so the + lead narrative never names or rates demoted content, its severity digest and + excerpts exclude it, and its window total and trend cover real feedback. A + window with only demoted content yields no Yhteenveto (the moderation count + still shows). Server-side in `ReportService`, so every view and the offline + snapshot inherit it. ## Consequences diff --git a/src/FeedbackIntelligence.Api/Analysis/ReportService.cs b/src/FeedbackIntelligence.Api/Analysis/ReportService.cs index 66e2ea1..4a770e1 100644 --- a/src/FeedbackIntelligence.Api/Analysis/ReportService.cs +++ b/src/FeedbackIntelligence.Api/Analysis/ReportService.cs @@ -262,25 +262,41 @@ int DemotedRank(string category) // same locked prompt + citation grounding + action bounding as any // theme narrative — the scope line in the data block is the only // difference. Falls back deterministically like everything else. + // + // The Yhteenveto is the TOP of the page, so it summarizes REAL feedback + // ONLY (ADR-0033 §3): demoted categories (rasismi/asiaton) are unrated — + // excluded from every severity/sentiment aggregate (ADR-0032) and carried + // behind the moderation count, never named or rated in the lead summary. + // Synthesizing over the RATED items keeps the demoted content out of the + // narrative, the severities digest, the excerpts, the window total, and the + // trend alike — otherwise the model is fed a slur's "critical" severity and + // dutifully writes it into the Yhteenveto ("'' (korkea)"). The + // reportSentiment mix already excludes it. A window that is ONLY demoted + // content has no rated feedback to summarize: Overall stays null and the + // moderation view carries the count. ReportTheme? overall = null; - if (liveSummary && structured.Count > 0) + if (liveSummary) { - var scope = ReportText.WholeWindowScope(lang); - var overallDirection = ComputeDirection(structured, fromIso, toIso, opts.MinItemsForTrend, opts.TrendSignificanceZ); - var overallDirectionLabel = ReportText.DirectionLabel(overallDirection, lang); - var synthesized = await SynthesizeThemeAsync(scope, structured, overallDirectionLabel, state, ct); - // Category "overall" is a REPORT-LEVEL key, deliberately outside any - // domain vocabulary; views render Overall by its own slot, never via - // a category-label lookup. Sources stay empty — the items live in - // the category sections. - overall = synthesized is { } ok - ? new ReportTheme("overall", ok.Title, ok.Narrative, structured.Count, overallDirection, - overallDirectionLabel, structured.Select(i => i.Id).ToList(), true, [], - structured.Count(i => i.NeedsReview), SentimentCounts: reportSentiment) - : new ReportTheme("overall", FallbackTitle(scope, structured), - FallbackNarrative(structured, overallDirectionLabel, lang), structured.Count, overallDirection, - overallDirectionLabel, structured.Select(i => i.Id).ToList(), false, [], - structured.Count(i => i.NeedsReview), SentimentCounts: reportSentiment); + var rated = structured.Where(i => !demotedCats.Contains(i.Structure!.Category)).ToList(); + if (rated.Count > 0) + { + var scope = ReportText.WholeWindowScope(lang); + var overallDirection = ComputeDirection(rated, fromIso, toIso, opts.MinItemsForTrend, opts.TrendSignificanceZ); + var overallDirectionLabel = ReportText.DirectionLabel(overallDirection, lang); + var synthesized = await SynthesizeThemeAsync(scope, rated, overallDirectionLabel, state, ct); + // Category "overall" is a REPORT-LEVEL key, deliberately outside any + // domain vocabulary; views render Overall by its own slot, never via + // a category-label lookup. Sources stay empty — the items live in + // the category sections. + overall = synthesized is { } ok + ? new ReportTheme("overall", ok.Title, ok.Narrative, rated.Count, overallDirection, + overallDirectionLabel, rated.Select(i => i.Id).ToList(), true, [], + rated.Count(i => i.NeedsReview), SentimentCounts: reportSentiment) + : new ReportTheme("overall", FallbackTitle(scope, rated), + FallbackNarrative(rated, overallDirectionLabel, lang), rated.Count, overallDirection, + overallDirectionLabel, rated.Select(i => i.Id).ToList(), false, [], + rated.Count(i => i.NeedsReview), SentimentCounts: reportSentiment); + } } var report = new ManagementReport( diff --git a/tests/FeedbackIntelligence.Api.Tests/ReportServiceTests.cs b/tests/FeedbackIntelligence.Api.Tests/ReportServiceTests.cs index e540c35..c9d6c8c 100644 --- a/tests/FeedbackIntelligence.Api.Tests/ReportServiceTests.cs +++ b/tests/FeedbackIntelligence.Api.Tests/ReportServiceTests.cs @@ -574,6 +574,39 @@ public async Task DemotedCategory_IsUnrated_SuppressesSentiment_AndExcludesFromM Assert.Equal(1, report.SentimentCounts!.GetValueOrDefault("negative")); } + [Fact] + public async Task SummaryMode_Overall_ExcludesDemotedContent_FromDigestCountAndTrend() + { + // ADR-0032 / ADR-0033 §3: the whole-window Yhteenveto is the LEAD summary and + // must reflect RATED feedback only. A demoted (rasismi/asiaton) item must never + // reach the synthesis model with its severity, nor be counted in the window + // total — otherwise the summary names hostile content and rates it, e.g. + // "'' (korkea)", and the severity distribution it recites includes the + // demoted item. Regression for that leak. + await _store.InsertAsync(ItemIn("maito-1", "2026-06-19T10:00:00.0000000+00:00", "maito_kylma", "tuoreus", "high"), CancellationToken.None); + await _store.InsertAsync(ItemIn("maito-2", "2026-06-20T10:00:00.0000000+00:00", "maito_kylma", "tuoreus", "low"), CancellationToken.None); + // A demoted item the model rated "critical" — the Yhteenveto must not see it. + await _store.InsertAsync(ItemIn("asia-1", "2026-06-21T10:00:00.0000000+00:00", "asiaton", "loukkaus", "critical"), CancellationToken.None); + + var llm = new CapturingScriptedChatClient( + """{"title": "Yleiskatsaus", "narrative": "Asiakkaat puhuvat tuoreudesta.", "citedIds": ["maito-1"]}"""); + + var report = await CreateService(llm) + .GenerateAsync(WindowFrom, WindowTo, CancellationToken.None, liveSummary: true); + + Assert.NotNull(report.Overall); + // Window total and grounding cover the two rated items only, never the demoted one. + Assert.Equal(2, report.Overall!.Count); + Assert.Contains("maito-1", report.Overall.FeedbackIds); + Assert.DoesNotContain("asia-1", report.Overall.FeedbackIds); + + // The synthesis digest the model actually saw carried neither the demoted item's + // id/excerpt nor its "critical" severity — only the rated items' high/low. + Assert.DoesNotContain("asia-1", llm.LastPrompt); + Assert.DoesNotContain("critical", llm.LastPrompt); + Assert.Contains("maito-1", llm.LastPrompt); + } + [Fact] public async Task Snapshot_PersistedOnlyOnOptIn_NotOnEphemeralView() { @@ -695,6 +728,31 @@ public void Dispose() } } + // Same fixed reply on every call, but captures the last prompt the model saw — + // used to assert what did (and did NOT) enter the synthesis data block, e.g. that + // a demoted item's excerpt/severity never reaches the Yhteenveto digest. + private sealed class CapturingScriptedChatClient(string response) : IChatClient + { + public string LastPrompt { get; private set; } = ""; + + public Task GetResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + LastPrompt = string.Join("\n", messages.Select(m => m.Text)); + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, response))); + } + + public IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() + { + } + } + private sealed class ThrowingChatClient : IChatClient { public Task GetResponseAsync(