Add a reject class for OOD detection

Scripts/gen_ood.py

@@ -0,0 +1,134 @@
+"""Generate an out-of-distribution (OOD) "none" class for the reject-class fix.
+
+routelet is overconfident: shown only the five valid intents, it confidently
+labels gibberish as one of them. Training it on a sixth "none" class of OOD /
+garbled text teaches it to actively flag input it shouldn't act on, which is a
+far more reliable signal than max-softmax confidence.
+
+This produces varied synthetic OOD across several families (gibberish, char
+salad, other languages, off-domain prose, number/symbol noise, ASR disfluency)
+so the model learns "weird" broadly rather than one narrow pattern. Output is
+split into a training file and a frozen eval file, with no overlap between them
+and no overlap with the hand-written report probe set (report/ood_probe.txt),
+so the eval stays honest.
+
+Run: uv run python Scripts/gen_ood.py
+"""
+
+from __future__ import annotations
+
+import json
+import random
+from pathlib import Path
+
+PROJECT_ROOT = Path(__file__).parent.parent
+TRAIN_OUT = PROJECT_ROOT / "data" / "none.jsonl"
+EVAL_OUT = PROJECT_ROOT / "evals" / "ood_holdout.jsonl"
+PROBE = PROJECT_ROOT / "report" / "ood_probe.txt"
+EVAL_FRACTION = 0.2
+SEED = 11
+
+_CV = "bcdfghjklmnpqrstvwz"
+_V = "aeiou"
+
+FOREIGN = [
+    "quelle heure est il maintenant", "ich moechte ein kaltes bier",
+    "donde esta la biblioteca por favor", "wo ist der naechste bahnhof",
+    "je voudrais un cafe au lait", "como se llama este lugar",
+    "il fait tres froid aujourd hui", "dov e la stazione dei treni",
+    "watashi wa nihongo ga hanasemasen", "guten morgen wie geht es ihnen",
+    "merci beaucoup pour votre aide", "no entiendo lo que dices",
+]
+OFF_DOMAIN = [
+    "the mitochondria is the powerhouse of the cell",
+    "photosynthesis converts sunlight into glucose and oxygen",
+    "the treaty of westphalia was signed in sixteen forty eight",
+    "ribosomes translate messenger rna into proteins",
+    "the french revolution began in seventeen eighty nine",
+    "water boils at one hundred degrees celsius at sea level",
+    "the mariana trench is the deepest part of the ocean",
+    "shakespeare wrote romeo and juliet in the fifteen nineties",
+    "the speed of light is roughly three hundred thousand kilometers per second",
+    "tectonic plates drift a few centimeters every year",
+    "the human heart beats about a hundred thousand times a day",
+    "jupiter is the largest planet in the solar system",
+    "the great wall of china stretches thousands of miles",
+    "honeybees communicate the location of flowers by dancing",
+    "the renaissance began in florence in the fourteenth century",
+    "a group of crows is called a murder",
+    "mount everest grows a little taller each year",
+    "the amazon rainforest produces a fifth of the world oxygen",
+]
+DISFLUENCY = [
+    "uh um like you know i mean", "er hmm well so anyway", "uhhh ahh ok ok ok",
+    "yeah yeah no no wait", "hmm let me think uhh", "so like basically um",
+    "wait what no hold on", "errr i dunno maybe sorta",
+]
+SYMBOLS = ["!@#$ %^&* ()_+", "<<<>>> {}{}{} []", "??? ... ;;; :::", "~~~ === +++ ---"]
+
+
+def _gibberish_word(rng: random.Random) -> str:
+    syl = rng.randint(1, 3)
+    return "".join(rng.choice(_CV) + rng.choice(_V) for _ in range(syl)) + (
+        rng.choice(_CV) if rng.random() < 0.4 else ""
+    )
+
+
+def _gibberish(rng: random.Random) -> str:
+    return " ".join(_gibberish_word(rng) for _ in range(rng.randint(2, 6)))
+
+
+def _char_salad(rng: random.Random) -> str:
+    n = rng.randint(8, 24)
+    return "".join(rng.choice(_CV + _V + "    ") for _ in range(n)).strip()
+
+
+def _numbers(rng: random.Random) -> str:
+    parts = [str(rng.randint(0, 9999)) for _ in range(rng.randint(2, 5))]
+    return " ".join(parts)
+
+
+def generate(rng: random.Random) -> list[str]:
+    out: set[str] = set()
+    # ~150 gibberish + ~80 char salad + ~60 numbers, plus the fixed pools, each
+    # variant uniqued by content.
+    for _ in range(150):
+        out.add(_gibberish(rng))
+    for _ in range(80):
+        out.add(_char_salad(rng))
+    for _ in range(60):
+        out.add(_numbers(rng))
+    out.update(FOREIGN)
+    out.update(OFF_DOMAIN)
+    out.update(DISFLUENCY)
+    out.update(SYMBOLS)
+    return sorted(out)
+
+
+def main() -> None:
+    rng = random.Random(SEED)
+    probe = set()
+    if PROBE.exists():
+        probe = {
+            ln.strip().lower()
+            for ln in PROBE.read_text().splitlines()
+            if ln.strip() and not ln.lstrip().startswith("#")
+        }
+
+    rows = [r for r in generate(rng) if r.strip() and r.lower() not in probe]
+    rng.shuffle(rows)
+    n_eval = int(len(rows) * EVAL_FRACTION)
+    eval_rows, train_rows = rows[:n_eval], rows[n_eval:]
+
+    for path, items in ((TRAIN_OUT, train_rows), (EVAL_OUT, eval_rows)):
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with path.open("w") as f:
+            for text in items:
+                f.write(json.dumps({"text": text, "intent": "none"}) + "\n")
+
+    print(f"generated {len(rows)} OOD rows ({len(train_rows)} train -> {TRAIN_OUT.name}, "
+          f"{len(eval_rows)} eval -> {EVAL_OUT.name})")
+
+
+if __name__ == "__main__":
+    main()

data/none.jsonl.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: 6272095238663d7780e1ea6655858a73
+  size: 13554
+  hash: md5
+  path: none.jsonl

docs/taxonomy.md

@@ -45,3 +45,15 @@ If a command still fits more than one after the rules above, pick the first matc
 `agent` > `memory` > `integration` > `find_action` > `chat`
 
 `chat` is the default. If nothing else clearly applies, it is `chat`.
+
+## Reject class (`none`)
+
+A sixth label, `none`, sits outside the five routing intents. It is **not** a
+command type: it marks out-of-distribution or garbled input the router should
+not act on (gibberish, other languages, off-domain prose, ASR noise). It exists
+because the model is otherwise overconfident, shown only valid commands, it
+labels junk as one of the five at ~98% confidence, so a confidence gate can't
+catch it. Training on a `none` class of generated OOD (see `Scripts/gen_ood.py`)
+lets the model say "I don't know" directly. Aegis treats a `none` prediction as
+"defer to Claude", never as a routing target. `none` is never hand-labeled or
+emitted by the Claude teacher; it is learned only from generated OOD data.

evals/ood_holdout.jsonl

@@ -0,0 +1,65 @@
+{"text": "tahurol volin ne zo leq mehatu", "intent": "none"}
+{"text": "duj ru pado kijo zatacod hedet", "intent": "none"}
+{"text": "leme zaha falibuv bediv koqi foqoluz", "intent": "none"}
+{"text": "mount everest grows a little taller each year", "intent": "none"}
+{"text": "de du fipo kawimo rojaso vevig", "intent": "none"}
+{"text": "mizo sodaf", "intent": "none"}
+{"text": "u u ccqcvu", "intent": "none"}
+{"text": "2265 5223", "intent": "none"}
+{"text": "dijebas cap zedigo vuguca mabu dine", "intent": "none"}
+{"text": "wo ist der naechste bahnhof", "intent": "none"}
+{"text": "ti ru higi", "intent": "none"}
+{"text": "fuhi wepu ze jubedez", "intent": "none"}
+{"text": "1543 7942 1301", "intent": "none"}
+{"text": "hicocuw wikewuh lenokov ker tijik", "intent": "none"}
+{"text": "ta wo letor vife le ceb", "intent": "none"}
+{"text": "kutuve qemu titonu", "intent": "none"}
+{"text": "nege ceke sif", "intent": "none"}
+{"text": "gkr ozgi bi oah", "intent": "none"}
+{"text": "u  vkokj", "intent": "none"}
+{"text": "nidilo gezu win gabipe hewa", "intent": "none"}
+{"text": "8606 7446", "intent": "none"}
+{"text": "er hmm well so anyway", "intent": "none"}
+{"text": "uhhh ahh ok ok ok", "intent": "none"}
+{"text": "buka zil fufu japaz", "intent": "none"}
+{"text": "taw vutejeh ma", "intent": "none"}
+{"text": "jaz honujo cofuj wuzobe hose", "intent": "none"}
+{"text": "can reno kij", "intent": "none"}
+{"text": "dov e la stazione dei treni", "intent": "none"}
+{"text": "co subipo bi", "intent": "none"}
+{"text": "fpwmcgsf uls fk", "intent": "none"}
+{"text": "tila alq sqqrzgvwrwkg", "intent": "none"}
+{"text": "fin wuruza ducoho tu ro", "intent": "none"}
+{"text": "honeybees communicate the location of flowers by dancing", "intent": "none"}
+{"text": "8547 4786 9035", "intent": "none"}
+{"text": "temi vijesi", "intent": "none"}
+{"text": "wukugac nuvizov qit sanosi cetuge cakovew", "intent": "none"}
+{"text": "9906 4144 4881 7030 9759", "intent": "none"}
+{"text": "nuzeso qo", "intent": "none"}
+{"text": "vew samemad", "intent": "none"}
+{"text": "johehet miripu kineve fis", "intent": "none"}
+{"text": "uta eiwpi wld iwocrtu", "intent": "none"}
+{"text": "5457 4738 2310 7407 4203", "intent": "none"}
+{"text": "hucaq dan cufemef", "intent": "none"}
+{"text": "hopufo wa tomadid monigen", "intent": "none"}
+{"text": "762 5613 7657 3758 3367", "intent": "none"}
+{"text": "jepap hobac fiweh dufu teto la", "intent": "none"}
+{"text": "kdkbg bhtzrrpwnm", "intent": "none"}
+{"text": "5708 5299 3978 998", "intent": "none"}
+{"text": "jibocum jupij", "intent": "none"}
+{"text": "6556 1530 4755", "intent": "none"}
+{"text": "lbwwqwkosgotkjft m", "intent": "none"}
+{"text": "stbsolcevbsifnuhornu", "intent": "none"}
+{"text": "qofef mugaput tesi kej paheze nifi", "intent": "none"}
+{"text": "putefoj ta", "intent": "none"}
+{"text": "fq  wetfpq lvq  o ugi", "intent": "none"}
+{"text": "lelu corobut teqica kefe siha", "intent": "none"}
+{"text": "ihw wakmf", "intent": "none"}
+{"text": "sofod pu fomufen", "intent": "none"}
+{"text": "ssi wofhbqk", "intent": "none"}
+{"text": "sovuje hasiw cuqohuv cak bonoze", "intent": "none"}
+{"text": "7386 3818", "intent": "none"}
+{"text": "lwfsloztg", "intent": "none"}
+{"text": "5690 8572 2919 8264", "intent": "none"}
+{"text": "6773 9451 7357", "intent": "none"}
+{"text": "d   qgeb", "intent": "none"}

report/metrics.json

@@ -6,11 +6,17 @@
   "setfit": {
     "accuracy": 0.925
   },
-  "gate": {
-    "threshold": 0.55,
-    "in_distribution_kept_share": 1.0,
-    "ood_deferred_share": 0.067,
-    "kept_accuracy": 0.925
+  "ood": {
+    "probe_n": 30,
+    "confidence_gate": {
+      "threshold": 0.95,
+      "ood_caught": 0.167,
+      "real_deferred": 0.02
+    },
+    "reject_class": {
+      "ood_caught": 0.8,
+      "real_deferred": 0.0
+    }
   },
   "haiku": {
     "model": "claude-haiku-4-5",