From 5a74ca2f69faed56ef2dbc4463a18d8484fc600e Mon Sep 17 00:00:00 2001 From: Rockford Lhotka Date: Tue, 21 Jul 2026 23:20:14 -0500 Subject: [PATCH] Add dream-tunable balancedFloorKeywords to floor tool-intent queries at Balanced (#486) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Short user queries that need a tool (e.g. "what's on my todo list?") score near-zero and route Low, where the small model fails to pick the right MCP tool under heavy injected context. The dream detects this (panicEscalation) but had no lever: topic words in highSignalKeywords over-route to High and are stripped by the TopicBlocklist; threshold nudges can't move a ~0.0 query. Add a dream-tunable balancedFloorKeywords list. When a user-message prompt matches a floor keyword and the computed tier is Low, escalate to Balanced (never High). Exempt from the TopicBlocklist by design — it holds topic/tool words. Seeded empty; the dream owns it, so behavior is unchanged until populated. - TierSelectorConfig: nullable BalancedFloorKeywords property - KeywordTierSelector: compiled default (empty), merge (auto-exempt from blocklist since list name lacks "high"), Low->Balanced override in ClassifyCore, floor count in reload log - routing-dream.md: document the lever as job 4; keep the reject-topic-words rule scoped to highSignalKeywords only - Tests: escalation, subagent no-op, never-High, non-match stays Low, blocklist exemption Co-Authored-By: Claude Opus 4.8 (1M context) --- src/RockBot.Agent/agent/routing-dream.md | 34 ++++- .../TierSelectorConfig.cs | 9 ++ src/RockBot.Llm/KeywordTierSelector.cs | 36 ++++- .../KeywordTierSelectorTests.cs | 138 ++++++++++++++++++ 4 files changed, 210 insertions(+), 7 deletions(-) diff --git a/src/RockBot.Agent/agent/routing-dream.md b/src/RockBot.Agent/agent/routing-dream.md index 0172519e..77f9829a 100644 --- a/src/RockBot.Agent/agent/routing-dream.md +++ b/src/RockBot.Agent/agent/routing-dream.md @@ -27,7 +27,7 @@ directive and analyzer have drifted and you cannot reliably interpret the input. - **`fallbackExcludedCount`** — fallback-triggered entries are already excluded from clusters, flagged clusters, and keyword candidates; do not filter them yourself -## Your three jobs +## Your four jobs ### 1. Validate flagged clusters @@ -36,6 +36,10 @@ For each entry in `flaggedClusters`: - **panicEscalation** — Low tier with avg tool calls ≥ 3. The model likely struggled. Check `samplePrompt` and `count`: if this looks like a real recurring shape (not a fluke), the cluster's `alternateTier` (Balanced) is the correct routing direction. + When such a cluster is a **short user query naming a tool/topic** (e.g. "what's on my + todo list?", "any new calendar events?") that scores near-zero and routes Low, threshold + nudges can't move it and topic words can't go in `highSignalKeywords`. Use the + **`config.balancedFloorKeywords`** lever instead — see job 4. - **tokenSurprise** — Low complexity score but high post-injection tokens. **Informational only** — DO NOT adjust thresholds for this. Post-injection size is an orchestration-layer concern, not a routing-quality signal. A trivial prompt like "what time is it?" legitimately @@ -98,6 +102,31 @@ Trivial guard fields (`trivialGuardCeiling`, `userOriginBias`) are available in but should only be touched when there is clear evidence of *false Low-tier routing* — never to widen Balanced. +### 4. Floor tool-intent queries at Balanced with `balancedFloorKeywords` + +Some short user queries **need a tool** but score near-zero and route **Low**, where the +small model fails to pick the right MCP tool under heavy injected context. These surface as +recurring **panicEscalation** clusters whose `samplePrompt` is a brief user request naming a +topic/tool (todo, calendar, email, reminders). Thresholds can't move a ~0.0 query, and the +topic word can't go in `highSignalKeywords` (it would over-route to expensive High and be +stripped by the topic blocklist anyway). + +The lever is **`config.balancedFloorKeywords`**: when a **user-message** prompt matches a +floor keyword **and** the computed tier is **Low**, the selector escalates it to **Balanced** +(cheap but tool-capable). It **never** escalates to High, and it does **not** affect subagent +traffic. This list is **exempt from the topic blocklist** — it is *meant* to hold topic/tool +words. Additions are merged with compiled defaults (add-only). + +Add a floor keyword only when a panicEscalation cluster shows a **recurring** tool-intent +shape (not a one-off). Pick the single recurring topic/tool word from the cluster's +`samplePrompt` (e.g. `todo`, `calendar`). Return your additions in the +`config.balancedFloorKeywords` array — additions only, merged with defaults. + +**Keep the distinction sharp:** floor words route **Low→Balanced** (cheap + tool-capable) +and **never** to High. The "reject topic words" rule in job 2 still applies to +`highSignalKeywords` only — topic words belong in `balancedFloorKeywords`, not in the +high-signal list. + ## Response format Return ONLY a JSON object: @@ -111,7 +140,8 @@ Return ONLY a JSON object: "lowCeiling": 0.20, "balancedCeiling": 0.46, "highSignalKeywords": ["additions only — merged with compiled defaults"], - "lowSignalKeywords": ["additions only — merged with compiled defaults"] + "lowSignalKeywords": ["additions only — merged with compiled defaults"], + "balancedFloorKeywords": ["tool/topic words that floor Low→Balanced — additions only"] }, "antiPatterns": [ { diff --git a/src/RockBot.Host.Abstractions/TierSelectorConfig.cs b/src/RockBot.Host.Abstractions/TierSelectorConfig.cs index 21b942fb..f3e9b7e2 100644 --- a/src/RockBot.Host.Abstractions/TierSelectorConfig.cs +++ b/src/RockBot.Host.Abstractions/TierSelectorConfig.cs @@ -30,6 +30,15 @@ public sealed class TierSelectorConfig /// public List? LowSignalKeywords { get; set; } + /// + /// Keywords that floor a user-message prompt at the Balanced tier when it would + /// otherwise route Low. Intended for tool/topic words (todo, calendar, ...) that + /// signal "this needs a tool" — the small Low model handles those unreliably. + /// Merged with compiled defaults (add-only). Escalates Low→Balanced only; never High. + /// Exempt from the high-signal TopicBlocklist. See issue #486. + /// + public List? BalancedFloorKeywords { get; set; } + /// /// Score threshold below which the trivial guard forces Low tier, regardless of /// dream-tuned thresholds. Prevents simple prompts from drifting into Balanced diff --git a/src/RockBot.Llm/KeywordTierSelector.cs b/src/RockBot.Llm/KeywordTierSelector.cs index 20ec9839..8fae4cc0 100644 --- a/src/RockBot.Llm/KeywordTierSelector.cs +++ b/src/RockBot.Llm/KeywordTierSelector.cs @@ -72,10 +72,16 @@ public sealed class KeywordTierSelector : ILlmTierSelector "tell me about", "show me", "look up", ]; + // ── Balanced-floor signals → floor a Low-routed user query at Balanced ─── + // Empty by default (add-only caveat): the dream owns this list and no + // un-retractable guesses are baked in. See issue #486. + private static readonly string[] DefaultBalancedFloorKeywords = []; + private static readonly EffectiveConfig Defaults = new( DefaultLowCeiling, DefaultBalancedCeiling, DefaultTrivialGuardCeiling, DefaultUserOriginBias, - DefaultHighSignalKeywords, DefaultLowSignalKeywords); + DefaultHighSignalKeywords, DefaultLowSignalKeywords, + DefaultBalancedFloorKeywords); // ── Code / math / multi-step markers ──────────────────────────────────── private static readonly Regex CodeBlockRegex = new( @@ -215,6 +221,19 @@ private TierClassification ClassifyCore(string promptText, TierRoutingContext? c tier = ModelTier.Balanced; } + // Balanced-floor override: a first-turn user query naming a known tool/topic + // (e.g. "what's on my todo list?") scores near-zero → Low, where the small model + // fails to select the right MCP tool. When the dream has learned such a floor + // keyword, escalate Low→Balanced (never High) so it lands on a cheap but + // tool-capable tier. Exempt from TopicBlocklist by design. See issue #486. + if (context?.Origin == "user-message" + && tier == ModelTier.Low + && config.BalancedFloorKeywords.Length > 0 + && config.BalancedFloorKeywords.Any(k => ContainsWholePhrase(lower, k))) + { + tier = ModelTier.Balanced; + } + return new TierClassification(tier, score, matchedHigh, matchedLow); } @@ -258,6 +277,10 @@ private EffectiveConfig TryLoad() // Merge dream keywords with compiled defaults (dream adds, never replaces). var highKeywords = MergeKeywords(DefaultHighSignalKeywords, dto.HighSignalKeywords, "highSignalKeywords"); var lowKeywords = MergeKeywords(DefaultLowSignalKeywords, dto.LowSignalKeywords, "lowSignalKeywords"); + // Balanced-floor list is exempt from the TopicBlocklist: SanitizeKeywords only + // applies the blocklist when the list name contains "high", so "balancedFloorKeywords" + // passes topic/tool words (todo, calendar, ...) through unfiltered — by design. + var floorKeywords = MergeKeywords(DefaultBalancedFloorKeywords, dto.BalancedFloorKeywords, "balancedFloorKeywords"); // Clamp dream-tuned thresholds to guardrail ranges. var lowCeiling = ClampThreshold(dto.LowCeiling, DefaultLowCeiling, MinLowCeiling, MaxLowCeiling, "lowCeiling"); @@ -271,16 +294,18 @@ private EffectiveConfig TryLoad() TrivialGuardCeiling: trivialGuard, UserOriginBias: originBias, HighSignalKeywords: highKeywords, - LowSignalKeywords: lowKeywords); + LowSignalKeywords: lowKeywords, + BalancedFloorKeywords: floorKeywords); _logger?.LogInformation( "KeywordTierSelector: reloaded config from {Path} " + "(lowCeiling={Low}, balancedCeiling={Balanced}, " + "trivialGuard={TrivialGuard}, userOriginBias={OriginBias}, " + - "highSignals={HighCount}, lowSignals={LowCount})", + "highSignals={HighCount}, lowSignals={LowCount}, floorKeywords={FloorCount})", _configPath, result.LowCeiling, result.BalancedCeiling, result.TrivialGuardCeiling, result.UserOriginBias, - result.HighSignalKeywords.Length, result.LowSignalKeywords.Length); + result.HighSignalKeywords.Length, result.LowSignalKeywords.Length, + result.BalancedFloorKeywords.Length); return result; } @@ -512,7 +537,8 @@ private sealed record EffectiveConfig( double TrivialGuardCeiling, double UserOriginBias, string[] HighSignalKeywords, - string[] LowSignalKeywords); + string[] LowSignalKeywords, + string[] BalancedFloorKeywords); private sealed record CachedConfig(EffectiveConfig Config, DateTime LoadedAt); } diff --git a/tests/RockBot.Llm.Tests/KeywordTierSelectorTests.cs b/tests/RockBot.Llm.Tests/KeywordTierSelectorTests.cs index a0742eb7..c8199112 100644 --- a/tests/RockBot.Llm.Tests/KeywordTierSelectorTests.cs +++ b/tests/RockBot.Llm.Tests/KeywordTierSelectorTests.cs @@ -860,6 +860,144 @@ public void Classify_LongBriefWithComplexityKeyword_StillRoutesHigh() "A long brief with genuine complexity keywords should still route High"); } + // ── Balanced-floor override (issue #486) ───────────────────────────────── + + [TestMethod] + public void BalancedFloor_UserOrigin_MatchEscalatesLowToBalanced() + { + var tempDir = Path.Combine(Path.GetTempPath(), "kts-floor-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + var config = """{"version":1,"balancedFloorKeywords":["todo"]}"""; + File.WriteAllText(Path.Combine(tempDir, "tier-selector.json"), config); + + var options = Options.Create(new AgentProfileOptions { BasePath = tempDir }); + var selector = new KeywordTierSelector(options, NullLogger.Instance); + + // A trivial tool-intent query that would otherwise route Low. + var result = selector.Classify( + "what's on my todo list?", + new TierRoutingContext(Origin: "user-message")); + + Assert.AreEqual(ModelTier.Balanced, result.Tier, + "A user-origin floor-keyword match should escalate Low to Balanced"); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [TestMethod] + public void BalancedFloor_SubagentOrigin_NoEffect() + { + var tempDir = Path.Combine(Path.GetTempPath(), "kts-floor-sub-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + var config = """{"version":1,"balancedFloorKeywords":["todo"]}"""; + File.WriteAllText(Path.Combine(tempDir, "tier-selector.json"), config); + + var options = Options.Create(new AgentProfileOptions { BasePath = tempDir }); + var selector = new KeywordTierSelector(options, NullLogger.Instance); + + // Same prompt/floor, subagent origin — the floor override must not fire. + var result = selector.Classify( + "what's on my todo list?", + new TierRoutingContext(Origin: "subagent")); + + Assert.AreEqual(ModelTier.Low, result.Tier, + "The balanced-floor override must not affect subagent traffic"); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [TestMethod] + public void BalancedFloor_NeverEscalatesToHigh() + { + var tempDir = Path.Combine(Path.GetTempPath(), "kts-floor-nohigh-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + var config = """{"version":1,"balancedFloorKeywords":["todo"]}"""; + File.WriteAllText(Path.Combine(tempDir, "tier-selector.json"), config); + + var options = Options.Create(new AgentProfileOptions { BasePath = tempDir }); + var selector = new KeywordTierSelector(options, NullLogger.Instance); + + var result = selector.Classify( + "what's on my todo list?", + new TierRoutingContext(Origin: "user-message")); + + Assert.AreNotEqual(ModelTier.High, result.Tier, + "The floor override only fires on Low and only sets Balanced — never High"); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [TestMethod] + public void BalancedFloor_NonMatchingTrivialChat_StaysLow() + { + var tempDir = Path.Combine(Path.GetTempPath(), "kts-floor-nomatch-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + var config = """{"version":1,"balancedFloorKeywords":["todo"]}"""; + File.WriteAllText(Path.Combine(tempDir, "tier-selector.json"), config); + + var options = Options.Create(new AgentProfileOptions { BasePath = tempDir }); + var selector = new KeywordTierSelector(options, NullLogger.Instance); + + // Trivial user chat that does NOT name the floor keyword — must stay Low. + var result = selector.Classify( + "what's the weather?", + new TierRoutingContext(Origin: "user-message")); + + Assert.AreEqual(ModelTier.Low, result.Tier, + "A trivial user prompt not matching any floor keyword must stay Low"); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + + [TestMethod] + public void BalancedFloor_TopicBlocklistExemption_FloorKeywordStillFires() + { + var tempDir = Path.Combine(Path.GetTempPath(), "kts-floor-exempt-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDir); + try + { + // "todo" is a blocklisted topic word (stripped from high-signal). It must + // survive in the balanced-floor list and still fire the override, proving + // the floor list is exempt from the TopicBlocklist. + var config = """{"version":1,"balancedFloorKeywords":["todo"]}"""; + File.WriteAllText(Path.Combine(tempDir, "tier-selector.json"), config); + + var options = Options.Create(new AgentProfileOptions { BasePath = tempDir }); + var selector = new KeywordTierSelector(options, NullLogger.Instance); + + var result = selector.Classify( + "what's on my todo list?", + new TierRoutingContext(Origin: "user-message")); + + Assert.AreEqual(ModelTier.Balanced, result.Tier, + "Blocklisted topic word 'todo' must not be stripped from the floor list"); + } + finally + { + Directory.Delete(tempDir, recursive: true); + } + } + [TestMethod] public void ActiveThreadOverride_MatchedHighKeyword_GateBlocksPromotion() {