feat(bench): fold typed registry capability fields into _model_caps

src/hal0/bench/planner.py

@@ -169,17 +169,51 @@ def _model_caps(m: dict[str, Any]) -> set[str]:
     "coder","tool-calling","mtp"). The roster selector wants the chat+coder
     roster, and "coder" lives in ``tags`` — so we match against the UNION of
     ``capabilities`` + ``tags`` (+ ``type``). ``caps`` is also accepted for the
-    synthetic registry entries the unit tests inject."""
+    synthetic registry entries the unit tests inject.
+
+    The freeform lists are unmaintained in practice: on a 49-model box only a
+    handful carry a semantic label, and this set is what stamps
+    ``identity.model.caps`` onto every benchmark record — which is where the
+    public leaderboard's capability pills come from (#1823). So the TYPED
+    fields are folded in too, following the ``defaults.mtp`` precedent below:
+    they are the surface an operator actually edits (``PATCH /api/models``),
+    so a typed edit should reach the roster without also hand-editing a
+    string list.
+
+    Only unambiguous positives are folded. Every typed flag here is
+    tri-state, and ``None`` means "unset / decided elsewhere", never "this
+    model lacks the capability" — so ``None`` and ``False`` both add nothing
+    rather than asserting an absence.
+
+    Widening this set is safe for selection: suites match with ``caps_any``,
+    so extra labels can only make a model match more selectors, never fewer.
+    """
     caps: set[str] = set()
     for field_name in ("caps", "capabilities", "tags"):
         caps.update(m.get(field_name) or [])
     if m.get("type"):
         caps.add(m["type"])
+    defaults = m.get("defaults") or {}
     # The type-tag retirement folds the pre-1.0 "mtp" tag into the typed
     # defaults.mtp field — surface it back into the match set so rosters
     # keyed on "mtp" keep selecting migrated rows.
-    if (m.get("defaults") or {}).get("mtp") is True:
+    if defaults.get("mtp") is True:
         caps.add("mtp")
+    # Vision is the projector, not the flag: registry/model.py documents
+    # defaults.vision=None as AUTO ("the mmproj sidecar loads whenever the
+    # model carries one") and True as "an explicit no-op affirmation". So the
+    # real signal is carrying an mmproj, and the only thing that revokes it is
+    # an explicit False (force-suppress).
+    if m.get("mmproj") and defaults.get("vision") is not False:
+        caps.add("vision")
+    # ModelCapabilities.tool_calling — the one typed-bool surface for the
+    # omni-router's tool-call gate.
+    if (m.get("capability_flags") or {}).get("tool_calling") is True:
+        caps.add("tool-calling")
+    # defaults.enable_thinking True = this model defaults to thinking ON, so
+    # it demonstrably supports it. None is global suppression, not incapacity.
+    if defaults.get("enable_thinking") is True:
+        caps.add("reasoning")
     return caps
 
 

tests/bench/test_planner.py

@@ -13,7 +13,7 @@
 
 import pytest
 
-from hal0.bench.planner import plan
+from hal0.bench.planner import _model_caps, plan
 from hal0.bench.schema import Host, Outcome, Record
 from hal0.bench.store import Store
 from hal0.bench.suites import suite_from_dict
@@ -258,3 +258,74 @@ def test_non_chat_cell_never_needs_a_tokenizer(self, store):
         cells = plan(_suite(), reg, store)  # module-level _suite() plans "tg"
         assert len(cells) == 1
         assert cells[0].tokenizer == ""
+
+
+class TestModelCapsTypedFields:
+    """`_model_caps` folds the TYPED registry fields in alongside the freeform
+    lists (#1823).
+
+    The freeform `capabilities`/`tags` are unmaintained in practice, while the
+    typed fields are what `PATCH /api/models` actually edits — and this set is
+    what stamps `identity.model.caps` onto every benchmark record, so it drives
+    the public leaderboard's capability pills.
+
+    Every typed flag is tri-state: None means "unset / decided elsewhere",
+    never "lacks the capability".
+    """
+
+    def test_freeform_lists_still_win_on_their_own(self):
+        assert _model_caps({"capabilities": ["chat"], "tags": ["coder"]}) == {"chat", "coder"}
+
+    def test_mmproj_presence_is_the_vision_signal(self):
+        # registry/model.py: defaults.vision=None is AUTO — the projector loads
+        # whenever the model carries one — and True is an explicit no-op.
+        assert "vision" in _model_caps({"mmproj": "/models/x/mmproj.gguf"})
+        assert "vision" in _model_caps(
+            {"mmproj": "/models/x/mmproj.gguf", "defaults": {"vision": True}}
+        )
+
+    def test_explicit_vision_false_suppresses_a_present_projector(self):
+        caps = _model_caps({"mmproj": "/models/x/mmproj.gguf", "defaults": {"vision": False}})
+        assert "vision" not in caps
+
+    def test_no_projector_means_no_vision_however_the_flag_reads(self):
+        assert "vision" not in _model_caps({"defaults": {"vision": True}})
+
+    def test_tool_calling_flag_folds_in(self):
+        assert "tool-calling" in _model_caps({"capability_flags": {"tool_calling": True}})
+
+    def test_enable_thinking_true_implies_reasoning(self):
+        assert "reasoning" in _model_caps({"defaults": {"enable_thinking": True}})
+
+    @pytest.mark.parametrize("value", [None, False])
+    def test_tri_state_none_and_false_assert_nothing(self, value):
+        caps = _model_caps(
+            {
+                "capability_flags": {"tool_calling": value},
+                "defaults": {"enable_thinking": value, "mtp": value},
+            }
+        )
+        assert caps == set()
+
+    def test_missing_tables_do_not_explode(self):
+        assert _model_caps({}) == set()
+        assert _model_caps({"defaults": None, "capability_flags": None}) == set()
+
+    def test_typed_and_freeform_union_rather_than_replace(self):
+        caps = _model_caps(
+            {
+                "capabilities": ["chat"],
+                "tags": ["coder"],
+                "mmproj": "/m/mmproj.gguf",
+                "capability_flags": {"tool_calling": True},
+                "defaults": {"mtp": True, "enable_thinking": True},
+            }
+        )
+        assert caps == {"chat", "coder", "vision", "tool-calling", "mtp", "reasoning"}
+
+    def test_widening_can_only_add_caps_any_matches(self):
+        """Suites match with caps_any, so a wider set never de-selects a model
+        that already matched — the property that makes this safe to land."""
+        narrow = _model_caps({"capabilities": ["chat"]})
+        wide = _model_caps({"capabilities": ["chat"], "capability_flags": {"tool_calling": True}})
+        assert narrow <= wide