Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion docs/decisions/0033-operational-alerts-moderation-view.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,13 @@ it should be recognized but never lead the page.
"Palautteet aiheittain" category chart and the entries tile — the top of the
page reflects **real feedback only**; the moderation disclosure carries the
conduct count. (Severity and sentiment already excluded the unrated categories
per ADR-0032.)
per ADR-0032.) This includes the whole-window **Yhteenveto** (the live-summary
`Overall`, ADR-0026): it is synthesized over the **rated** items only, so the
lead narrative never names or rates demoted content, its severity digest and
excerpts exclude it, and its window total and trend cover real feedback. A
window with only demoted content yields no Yhteenveto (the moderation count
still shows). Server-side in `ReportService`, so every view and the offline
snapshot inherit it.

## Consequences

Expand Down
50 changes: 33 additions & 17 deletions src/FeedbackIntelligence.Api/Analysis/ReportService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -262,25 +262,41 @@ int DemotedRank(string category)
// same locked prompt + citation grounding + action bounding as any
// theme narrative — the scope line in the data block is the only
// difference. Falls back deterministically like everything else.
//
// The Yhteenveto is the TOP of the page, so it summarizes REAL feedback
// ONLY (ADR-0033 §3): demoted categories (rasismi/asiaton) are unrated —
// excluded from every severity/sentiment aggregate (ADR-0032) and carried
// behind the moderation count, never named or rated in the lead summary.
// Synthesizing over the RATED items keeps the demoted content out of the
// narrative, the severities digest, the excerpts, the window total, and the
// trend alike — otherwise the model is fed a slur's "critical" severity and
// dutifully writes it into the Yhteenveto ("'<slur>' (korkea)"). The
// reportSentiment mix already excludes it. A window that is ONLY demoted
// content has no rated feedback to summarize: Overall stays null and the
// moderation view carries the count.
ReportTheme? overall = null;
if (liveSummary && structured.Count > 0)
if (liveSummary)
{
var scope = ReportText.WholeWindowScope(lang);
var overallDirection = ComputeDirection(structured, fromIso, toIso, opts.MinItemsForTrend, opts.TrendSignificanceZ);
var overallDirectionLabel = ReportText.DirectionLabel(overallDirection, lang);
var synthesized = await SynthesizeThemeAsync(scope, structured, overallDirectionLabel, state, ct);
// Category "overall" is a REPORT-LEVEL key, deliberately outside any
// domain vocabulary; views render Overall by its own slot, never via
// a category-label lookup. Sources stay empty — the items live in
// the category sections.
overall = synthesized is { } ok
? new ReportTheme("overall", ok.Title, ok.Narrative, structured.Count, overallDirection,
overallDirectionLabel, structured.Select(i => i.Id).ToList(), true, [],
structured.Count(i => i.NeedsReview), SentimentCounts: reportSentiment)
: new ReportTheme("overall", FallbackTitle(scope, structured),
FallbackNarrative(structured, overallDirectionLabel, lang), structured.Count, overallDirection,
overallDirectionLabel, structured.Select(i => i.Id).ToList(), false, [],
structured.Count(i => i.NeedsReview), SentimentCounts: reportSentiment);
var rated = structured.Where(i => !demotedCats.Contains(i.Structure!.Category)).ToList();
if (rated.Count > 0)
{
var scope = ReportText.WholeWindowScope(lang);
var overallDirection = ComputeDirection(rated, fromIso, toIso, opts.MinItemsForTrend, opts.TrendSignificanceZ);
var overallDirectionLabel = ReportText.DirectionLabel(overallDirection, lang);
var synthesized = await SynthesizeThemeAsync(scope, rated, overallDirectionLabel, state, ct);
// Category "overall" is a REPORT-LEVEL key, deliberately outside any
// domain vocabulary; views render Overall by its own slot, never via
// a category-label lookup. Sources stay empty — the items live in
// the category sections.
overall = synthesized is { } ok
? new ReportTheme("overall", ok.Title, ok.Narrative, rated.Count, overallDirection,
overallDirectionLabel, rated.Select(i => i.Id).ToList(), true, [],
rated.Count(i => i.NeedsReview), SentimentCounts: reportSentiment)
: new ReportTheme("overall", FallbackTitle(scope, rated),
FallbackNarrative(rated, overallDirectionLabel, lang), rated.Count, overallDirection,
overallDirectionLabel, rated.Select(i => i.Id).ToList(), false, [],
rated.Count(i => i.NeedsReview), SentimentCounts: reportSentiment);
}
}

var report = new ManagementReport(
Expand Down
58 changes: 58 additions & 0 deletions tests/FeedbackIntelligence.Api.Tests/ReportServiceTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -564,7 +564,7 @@
var asiaton = report.Themes.Single(t => t.Category == "asiaton");
Assert.True(asiaton.Unrated);
Assert.All(asiaton.Sources, s => Assert.Null(s.Sentiment)); // no good/bad on demoted content
Assert.Empty(asiaton.SentimentCounts);

Check warning on line 567 in tests/FeedbackIntelligence.Api.Tests/ReportServiceTests.cs

View workflow job for this annotation

GitHub Actions / build + test (.NET 8)

Possible null reference argument for parameter 'collection' in 'void Assert.Empty(IEnumerable collection)'.

Check warning on line 567 in tests/FeedbackIntelligence.Api.Tests/ReportServiceTests.cs

View workflow job for this annotation

GitHub Actions / build + test (.NET 8)

Possible null reference argument for parameter 'collection' in 'void Assert.Empty(IEnumerable collection)'.

var maito = report.Themes.Single(t => t.Category == "maito_kylma");
Assert.False(maito.Unrated);
Expand All @@ -574,6 +574,39 @@
Assert.Equal(1, report.SentimentCounts!.GetValueOrDefault("negative"));
}

[Fact]
public async Task SummaryMode_Overall_ExcludesDemotedContent_FromDigestCountAndTrend()
{
// ADR-0032 / ADR-0033 §3: the whole-window Yhteenveto is the LEAD summary and
// must reflect RATED feedback only. A demoted (rasismi/asiaton) item must never
// reach the synthesis model with its severity, nor be counted in the window
// total — otherwise the summary names hostile content and rates it, e.g.
// "'<slur>' (korkea)", and the severity distribution it recites includes the
// demoted item. Regression for that leak.
await _store.InsertAsync(ItemIn("maito-1", "2026-06-19T10:00:00.0000000+00:00", "maito_kylma", "tuoreus", "high"), CancellationToken.None);
await _store.InsertAsync(ItemIn("maito-2", "2026-06-20T10:00:00.0000000+00:00", "maito_kylma", "tuoreus", "low"), CancellationToken.None);
// A demoted item the model rated "critical" — the Yhteenveto must not see it.
await _store.InsertAsync(ItemIn("asia-1", "2026-06-21T10:00:00.0000000+00:00", "asiaton", "loukkaus", "critical"), CancellationToken.None);

var llm = new CapturingScriptedChatClient(
"""{"title": "Yleiskatsaus", "narrative": "Asiakkaat puhuvat tuoreudesta.", "citedIds": ["maito-1"]}""");

var report = await CreateService(llm)
.GenerateAsync(WindowFrom, WindowTo, CancellationToken.None, liveSummary: true);

Assert.NotNull(report.Overall);
// Window total and grounding cover the two rated items only, never the demoted one.
Assert.Equal(2, report.Overall!.Count);
Assert.Contains("maito-1", report.Overall.FeedbackIds);
Assert.DoesNotContain("asia-1", report.Overall.FeedbackIds);

// The synthesis digest the model actually saw carried neither the demoted item's
// id/excerpt nor its "critical" severity — only the rated items' high/low.
Assert.DoesNotContain("asia-1", llm.LastPrompt);
Assert.DoesNotContain("critical", llm.LastPrompt);
Assert.Contains("maito-1", llm.LastPrompt);
}

[Fact]
public async Task Snapshot_PersistedOnlyOnOptIn_NotOnEphemeralView()
{
Expand Down Expand Up @@ -695,6 +728,31 @@
}
}

// Same fixed reply on every call, but captures the last prompt the model saw —
// used to assert what did (and did NOT) enter the synthesis data block, e.g. that
// a demoted item's excerpt/severity never reaches the Yhteenveto digest.
private sealed class CapturingScriptedChatClient(string response) : IChatClient
{
public string LastPrompt { get; private set; } = "";

public Task<ChatResponse> GetResponseAsync(
IEnumerable<ChatMessage> messages, ChatOptions? options = null, CancellationToken cancellationToken = default)
{
LastPrompt = string.Join("\n", messages.Select(m => m.Text));
return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, response)));
}

public IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
IEnumerable<ChatMessage> messages, ChatOptions? options = null, CancellationToken cancellationToken = default)
=> throw new NotSupportedException();

public object? GetService(Type serviceType, object? serviceKey = null) => null;

public void Dispose()
{
}
}

private sealed class ThrowingChatClient : IChatClient
{
public Task<ChatResponse> GetResponseAsync(
Expand Down
Loading