From 1a915c6dcc1ab3dd0995eea4ff8e44064b836326 Mon Sep 17 00:00:00 2001 From: thinMint Date: Mon, 10 Aug 2026 04:53:36 -0400 Subject: [PATCH] feat(bench): fold typed registry capability fields into _model_caps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _model_caps matched only the freeform `capabilities`/`tags` lists, which are unmaintained in practice: across 49 installed models on a live box, only a handful carry a semantic label. That set stamps `identity.model.caps` onto every benchmark record, which is where the public leaderboard's capability pills come from — so a model whose capabilities are recorded in the TYPED fields showed no pill at all (#1823). The typed fields are the surface an operator actually edits via PATCH /api/models, so a typed edit should reach the roster without also hand-editing a parallel string list. Folded in following the existing defaults.mtp precedent: - vision <- carrying an mmproj, unless defaults.vision is explicitly False. registry/model.py documents defaults.vision=None as AUTO ("the sidecar loads whenever the model carries one") and True as an explicit no-op affirmation, so the projector — not the flag — is the real signal. - tool-calling <- capability_flags.tool_calling is True - reasoning <- defaults.enable_thinking is True Only unambiguous positives fold in. Every flag is tri-state and None means "unset / decided elsewhere", never "lacks the capability", so None and False both add nothing rather than asserting an absence. Safe for benchmark selection: suites match with caps_any, so a wider set can only make a model match more selectors, never fewer. The shipped suites select on ["chat", "coder"], which none of these labels touch. This is plumbing, not a fix for the coverage gap itself — on today's data it recovers exactly one model, because the typed fields are nearly as empty as the freeform ones (tool_calling set on 1 of 49, enable_thinking on 1). What it buys is that labelling from here on lands in one place and propagates. Note that caps are captured into records at run time, so the leaderboard only reflects a labelling pass after the affected models are re-benchmarked. Refs #1823 Co-Authored-By: Claude Opus 5 --- src/hal0/bench/planner.py | 38 ++++++++++++++++++- tests/bench/test_planner.py | 73 ++++++++++++++++++++++++++++++++++++- 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/src/hal0/bench/planner.py b/src/hal0/bench/planner.py index 43e91038f..52b1995d2 100644 --- a/src/hal0/bench/planner.py +++ b/src/hal0/bench/planner.py @@ -169,17 +169,51 @@ def _model_caps(m: dict[str, Any]) -> set[str]: "coder","tool-calling","mtp"). The roster selector wants the chat+coder roster, and "coder" lives in ``tags`` — so we match against the UNION of ``capabilities`` + ``tags`` (+ ``type``). ``caps`` is also accepted for the - synthetic registry entries the unit tests inject.""" + synthetic registry entries the unit tests inject. + + The freeform lists are unmaintained in practice: on a 49-model box only a + handful carry a semantic label, and this set is what stamps + ``identity.model.caps`` onto every benchmark record — which is where the + public leaderboard's capability pills come from (#1823). So the TYPED + fields are folded in too, following the ``defaults.mtp`` precedent below: + they are the surface an operator actually edits (``PATCH /api/models``), + so a typed edit should reach the roster without also hand-editing a + string list. + + Only unambiguous positives are folded. Every typed flag here is + tri-state, and ``None`` means "unset / decided elsewhere", never "this + model lacks the capability" — so ``None`` and ``False`` both add nothing + rather than asserting an absence. + + Widening this set is safe for selection: suites match with ``caps_any``, + so extra labels can only make a model match more selectors, never fewer. + """ caps: set[str] = set() for field_name in ("caps", "capabilities", "tags"): caps.update(m.get(field_name) or []) if m.get("type"): caps.add(m["type"]) + defaults = m.get("defaults") or {} # The type-tag retirement folds the pre-1.0 "mtp" tag into the typed # defaults.mtp field — surface it back into the match set so rosters # keyed on "mtp" keep selecting migrated rows. - if (m.get("defaults") or {}).get("mtp") is True: + if defaults.get("mtp") is True: caps.add("mtp") + # Vision is the projector, not the flag: registry/model.py documents + # defaults.vision=None as AUTO ("the mmproj sidecar loads whenever the + # model carries one") and True as "an explicit no-op affirmation". So the + # real signal is carrying an mmproj, and the only thing that revokes it is + # an explicit False (force-suppress). + if m.get("mmproj") and defaults.get("vision") is not False: + caps.add("vision") + # ModelCapabilities.tool_calling — the one typed-bool surface for the + # omni-router's tool-call gate. + if (m.get("capability_flags") or {}).get("tool_calling") is True: + caps.add("tool-calling") + # defaults.enable_thinking True = this model defaults to thinking ON, so + # it demonstrably supports it. None is global suppression, not incapacity. + if defaults.get("enable_thinking") is True: + caps.add("reasoning") return caps diff --git a/tests/bench/test_planner.py b/tests/bench/test_planner.py index e6cacd5d1..121aecaec 100644 --- a/tests/bench/test_planner.py +++ b/tests/bench/test_planner.py @@ -13,7 +13,7 @@ import pytest -from hal0.bench.planner import plan +from hal0.bench.planner import _model_caps, plan from hal0.bench.schema import Host, Outcome, Record from hal0.bench.store import Store from hal0.bench.suites import suite_from_dict @@ -258,3 +258,74 @@ def test_non_chat_cell_never_needs_a_tokenizer(self, store): cells = plan(_suite(), reg, store) # module-level _suite() plans "tg" assert len(cells) == 1 assert cells[0].tokenizer == "" + + +class TestModelCapsTypedFields: + """`_model_caps` folds the TYPED registry fields in alongside the freeform + lists (#1823). + + The freeform `capabilities`/`tags` are unmaintained in practice, while the + typed fields are what `PATCH /api/models` actually edits — and this set is + what stamps `identity.model.caps` onto every benchmark record, so it drives + the public leaderboard's capability pills. + + Every typed flag is tri-state: None means "unset / decided elsewhere", + never "lacks the capability". + """ + + def test_freeform_lists_still_win_on_their_own(self): + assert _model_caps({"capabilities": ["chat"], "tags": ["coder"]}) == {"chat", "coder"} + + def test_mmproj_presence_is_the_vision_signal(self): + # registry/model.py: defaults.vision=None is AUTO — the projector loads + # whenever the model carries one — and True is an explicit no-op. + assert "vision" in _model_caps({"mmproj": "/models/x/mmproj.gguf"}) + assert "vision" in _model_caps( + {"mmproj": "/models/x/mmproj.gguf", "defaults": {"vision": True}} + ) + + def test_explicit_vision_false_suppresses_a_present_projector(self): + caps = _model_caps({"mmproj": "/models/x/mmproj.gguf", "defaults": {"vision": False}}) + assert "vision" not in caps + + def test_no_projector_means_no_vision_however_the_flag_reads(self): + assert "vision" not in _model_caps({"defaults": {"vision": True}}) + + def test_tool_calling_flag_folds_in(self): + assert "tool-calling" in _model_caps({"capability_flags": {"tool_calling": True}}) + + def test_enable_thinking_true_implies_reasoning(self): + assert "reasoning" in _model_caps({"defaults": {"enable_thinking": True}}) + + @pytest.mark.parametrize("value", [None, False]) + def test_tri_state_none_and_false_assert_nothing(self, value): + caps = _model_caps( + { + "capability_flags": {"tool_calling": value}, + "defaults": {"enable_thinking": value, "mtp": value}, + } + ) + assert caps == set() + + def test_missing_tables_do_not_explode(self): + assert _model_caps({}) == set() + assert _model_caps({"defaults": None, "capability_flags": None}) == set() + + def test_typed_and_freeform_union_rather_than_replace(self): + caps = _model_caps( + { + "capabilities": ["chat"], + "tags": ["coder"], + "mmproj": "/m/mmproj.gguf", + "capability_flags": {"tool_calling": True}, + "defaults": {"mtp": True, "enable_thinking": True}, + } + ) + assert caps == {"chat", "coder", "vision", "tool-calling", "mtp", "reasoning"} + + def test_widening_can_only_add_caps_any_matches(self): + """Suites match with caps_any, so a wider set never de-selects a model + that already matched — the property that makes this safe to land.""" + narrow = _model_caps({"capabilities": ["chat"]}) + wide = _model_caps({"capabilities": ["chat"], "capability_flags": {"tool_calling": True}}) + assert narrow <= wide