diff --git a/Scripts/gen_ood.py b/Scripts/gen_ood.py new file mode 100644 index 0000000..0275e58 --- /dev/null +++ b/Scripts/gen_ood.py @@ -0,0 +1,134 @@ +"""Generate an out-of-distribution (OOD) "none" class for the reject-class fix. + +routelet is overconfident: shown only the five valid intents, it confidently +labels gibberish as one of them. Training it on a sixth "none" class of OOD / +garbled text teaches it to actively flag input it shouldn't act on, which is a +far more reliable signal than max-softmax confidence. + +This produces varied synthetic OOD across several families (gibberish, char +salad, other languages, off-domain prose, number/symbol noise, ASR disfluency) +so the model learns "weird" broadly rather than one narrow pattern. Output is +split into a training file and a frozen eval file, with no overlap between them +and no overlap with the hand-written report probe set (report/ood_probe.txt), +so the eval stays honest. + +Run: uv run python Scripts/gen_ood.py +""" + +from __future__ import annotations + +import json +import random +from pathlib import Path + +PROJECT_ROOT = Path(__file__).parent.parent +TRAIN_OUT = PROJECT_ROOT / "data" / "none.jsonl" +EVAL_OUT = PROJECT_ROOT / "evals" / "ood_holdout.jsonl" +PROBE = PROJECT_ROOT / "report" / "ood_probe.txt" +EVAL_FRACTION = 0.2 +SEED = 11 + +_CV = "bcdfghjklmnpqrstvwz" +_V = "aeiou" + +FOREIGN = [ + "quelle heure est il maintenant", "ich moechte ein kaltes bier", + "donde esta la biblioteca por favor", "wo ist der naechste bahnhof", + "je voudrais un cafe au lait", "como se llama este lugar", + "il fait tres froid aujourd hui", "dov e la stazione dei treni", + "watashi wa nihongo ga hanasemasen", "guten morgen wie geht es ihnen", + "merci beaucoup pour votre aide", "no entiendo lo que dices", +] +OFF_DOMAIN = [ + "the mitochondria is the powerhouse of the cell", + "photosynthesis converts sunlight into glucose and oxygen", + "the treaty of westphalia was signed in sixteen forty eight", + "ribosomes translate messenger rna into proteins", + "the french revolution began in seventeen eighty nine", + "water boils at one hundred degrees celsius at sea level", + "the mariana trench is the deepest part of the ocean", + "shakespeare wrote romeo and juliet in the fifteen nineties", + "the speed of light is roughly three hundred thousand kilometers per second", + "tectonic plates drift a few centimeters every year", + "the human heart beats about a hundred thousand times a day", + "jupiter is the largest planet in the solar system", + "the great wall of china stretches thousands of miles", + "honeybees communicate the location of flowers by dancing", + "the renaissance began in florence in the fourteenth century", + "a group of crows is called a murder", + "mount everest grows a little taller each year", + "the amazon rainforest produces a fifth of the world oxygen", +] +DISFLUENCY = [ + "uh um like you know i mean", "er hmm well so anyway", "uhhh ahh ok ok ok", + "yeah yeah no no wait", "hmm let me think uhh", "so like basically um", + "wait what no hold on", "errr i dunno maybe sorta", +] +SYMBOLS = ["!@#$ %^&* ()_+", "<<<>>> {}{}{} []", "??? ... ;;; :::", "~~~ === +++ ---"] + + +def _gibberish_word(rng: random.Random) -> str: + syl = rng.randint(1, 3) + return "".join(rng.choice(_CV) + rng.choice(_V) for _ in range(syl)) + ( + rng.choice(_CV) if rng.random() < 0.4 else "" + ) + + +def _gibberish(rng: random.Random) -> str: + return " ".join(_gibberish_word(rng) for _ in range(rng.randint(2, 6))) + + +def _char_salad(rng: random.Random) -> str: + n = rng.randint(8, 24) + return "".join(rng.choice(_CV + _V + " ") for _ in range(n)).strip() + + +def _numbers(rng: random.Random) -> str: + parts = [str(rng.randint(0, 9999)) for _ in range(rng.randint(2, 5))] + return " ".join(parts) + + +def generate(rng: random.Random) -> list[str]: + out: set[str] = set() + # ~150 gibberish + ~80 char salad + ~60 numbers, plus the fixed pools, each + # variant uniqued by content. + for _ in range(150): + out.add(_gibberish(rng)) + for _ in range(80): + out.add(_char_salad(rng)) + for _ in range(60): + out.add(_numbers(rng)) + out.update(FOREIGN) + out.update(OFF_DOMAIN) + out.update(DISFLUENCY) + out.update(SYMBOLS) + return sorted(out) + + +def main() -> None: + rng = random.Random(SEED) + probe = set() + if PROBE.exists(): + probe = { + ln.strip().lower() + for ln in PROBE.read_text().splitlines() + if ln.strip() and not ln.lstrip().startswith("#") + } + + rows = [r for r in generate(rng) if r.strip() and r.lower() not in probe] + rng.shuffle(rows) + n_eval = int(len(rows) * EVAL_FRACTION) + eval_rows, train_rows = rows[:n_eval], rows[n_eval:] + + for path, items in ((TRAIN_OUT, train_rows), (EVAL_OUT, eval_rows)): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w") as f: + for text in items: + f.write(json.dumps({"text": text, "intent": "none"}) + "\n") + + print(f"generated {len(rows)} OOD rows ({len(train_rows)} train -> {TRAIN_OUT.name}, " + f"{len(eval_rows)} eval -> {EVAL_OUT.name})") + + +if __name__ == "__main__": + main() diff --git a/data/none.jsonl.dvc b/data/none.jsonl.dvc new file mode 100644 index 0000000..8f5dd00 --- /dev/null +++ b/data/none.jsonl.dvc @@ -0,0 +1,5 @@ +outs: +- md5: 6272095238663d7780e1ea6655858a73 + size: 13554 + hash: md5 + path: none.jsonl diff --git a/docs/taxonomy.md b/docs/taxonomy.md index 741b0ad..808ea61 100644 --- a/docs/taxonomy.md +++ b/docs/taxonomy.md @@ -45,3 +45,15 @@ If a command still fits more than one after the rules above, pick the first matc `agent` > `memory` > `integration` > `find_action` > `chat` `chat` is the default. If nothing else clearly applies, it is `chat`. + +## Reject class (`none`) + +A sixth label, `none`, sits outside the five routing intents. It is **not** a +command type: it marks out-of-distribution or garbled input the router should +not act on (gibberish, other languages, off-domain prose, ASR noise). It exists +because the model is otherwise overconfident, shown only valid commands, it +labels junk as one of the five at ~98% confidence, so a confidence gate can't +catch it. Training on a `none` class of generated OOD (see `Scripts/gen_ood.py`) +lets the model say "I don't know" directly. Aegis treats a `none` prediction as +"defer to Claude", never as a routing target. `none` is never hand-labeled or +emitted by the Claude teacher; it is learned only from generated OOD data. diff --git a/evals/ood_holdout.jsonl b/evals/ood_holdout.jsonl new file mode 100644 index 0000000..7ea030a --- /dev/null +++ b/evals/ood_holdout.jsonl @@ -0,0 +1,65 @@ +{"text": "tahurol volin ne zo leq mehatu", "intent": "none"} +{"text": "duj ru pado kijo zatacod hedet", "intent": "none"} +{"text": "leme zaha falibuv bediv koqi foqoluz", "intent": "none"} +{"text": "mount everest grows a little taller each year", "intent": "none"} +{"text": "de du fipo kawimo rojaso vevig", "intent": "none"} +{"text": "mizo sodaf", "intent": "none"} +{"text": "u u ccqcvu", "intent": "none"} +{"text": "2265 5223", "intent": "none"} +{"text": "dijebas cap zedigo vuguca mabu dine", "intent": "none"} +{"text": "wo ist der naechste bahnhof", "intent": "none"} +{"text": "ti ru higi", "intent": "none"} +{"text": "fuhi wepu ze jubedez", "intent": "none"} +{"text": "1543 7942 1301", "intent": "none"} +{"text": "hicocuw wikewuh lenokov ker tijik", "intent": "none"} +{"text": "ta wo letor vife le ceb", "intent": "none"} +{"text": "kutuve qemu titonu", "intent": "none"} +{"text": "nege ceke sif", "intent": "none"} +{"text": "gkr ozgi bi oah", "intent": "none"} +{"text": "u vkokj", "intent": "none"} +{"text": "nidilo gezu win gabipe hewa", "intent": "none"} +{"text": "8606 7446", "intent": "none"} +{"text": "er hmm well so anyway", "intent": "none"} +{"text": "uhhh ahh ok ok ok", "intent": "none"} +{"text": "buka zil fufu japaz", "intent": "none"} +{"text": "taw vutejeh ma", "intent": "none"} +{"text": "jaz honujo cofuj wuzobe hose", "intent": "none"} +{"text": "can reno kij", "intent": "none"} +{"text": "dov e la stazione dei treni", "intent": "none"} +{"text": "co subipo bi", "intent": "none"} +{"text": "fpwmcgsf uls fk", "intent": "none"} +{"text": "tila alq sqqrzgvwrwkg", "intent": "none"} +{"text": "fin wuruza ducoho tu ro", "intent": "none"} +{"text": "honeybees communicate the location of flowers by dancing", "intent": "none"} +{"text": "8547 4786 9035", "intent": "none"} +{"text": "temi vijesi", "intent": "none"} +{"text": "wukugac nuvizov qit sanosi cetuge cakovew", "intent": "none"} +{"text": "9906 4144 4881 7030 9759", "intent": "none"} +{"text": "nuzeso qo", "intent": "none"} +{"text": "vew samemad", "intent": "none"} +{"text": "johehet miripu kineve fis", "intent": "none"} +{"text": "uta eiwpi wld iwocrtu", "intent": "none"} +{"text": "5457 4738 2310 7407 4203", "intent": "none"} +{"text": "hucaq dan cufemef", "intent": "none"} +{"text": "hopufo wa tomadid monigen", "intent": "none"} +{"text": "762 5613 7657 3758 3367", "intent": "none"} +{"text": "jepap hobac fiweh dufu teto la", "intent": "none"} +{"text": "kdkbg bhtzrrpwnm", "intent": "none"} +{"text": "5708 5299 3978 998", "intent": "none"} +{"text": "jibocum jupij", "intent": "none"} +{"text": "6556 1530 4755", "intent": "none"} +{"text": "lbwwqwkosgotkjft m", "intent": "none"} +{"text": "stbsolcevbsifnuhornu", "intent": "none"} +{"text": "qofef mugaput tesi kej paheze nifi", "intent": "none"} +{"text": "putefoj ta", "intent": "none"} +{"text": "fq wetfpq lvq o ugi", "intent": "none"} +{"text": "lelu corobut teqica kefe siha", "intent": "none"} +{"text": "ihw wakmf", "intent": "none"} +{"text": "sofod pu fomufen", "intent": "none"} +{"text": "ssi wofhbqk", "intent": "none"} +{"text": "sovuje hasiw cuqohuv cak bonoze", "intent": "none"} +{"text": "7386 3818", "intent": "none"} +{"text": "lwfsloztg", "intent": "none"} +{"text": "5690 8572 2919 8264", "intent": "none"} +{"text": "6773 9451 7357", "intent": "none"} +{"text": "d qgeb", "intent": "none"} diff --git a/report/confidence_histogram.png b/report/confidence_histogram.png deleted file mode 100644 index 60a7c50..0000000 Binary files a/report/confidence_histogram.png and /dev/null differ diff --git a/report/deferral_tradeoff.png b/report/deferral_tradeoff.png deleted file mode 100644 index 4a150c5..0000000 Binary files a/report/deferral_tradeoff.png and /dev/null differ diff --git a/report/metrics.json b/report/metrics.json index 576dbe1..4d741a1 100644 --- a/report/metrics.json +++ b/report/metrics.json @@ -6,11 +6,17 @@ "setfit": { "accuracy": 0.925 }, - "gate": { - "threshold": 0.55, - "in_distribution_kept_share": 1.0, - "ood_deferred_share": 0.067, - "kept_accuracy": 0.925 + "ood": { + "probe_n": 30, + "confidence_gate": { + "threshold": 0.95, + "ood_caught": 0.167, + "real_deferred": 0.02 + }, + "reject_class": { + "ood_caught": 0.8, + "real_deferred": 0.0 + } }, "haiku": { "model": "claude-haiku-4-5", diff --git a/report/ood_detection.png b/report/ood_detection.png new file mode 100644 index 0000000..fa58f2f Binary files /dev/null and b/report/ood_detection.png differ diff --git a/report/report.py b/report/report.py index b51aaae..e8fcc94 100644 --- a/report/report.py +++ b/report/report.py @@ -51,7 +51,7 @@ # Routelet's on-device confidence gate: below this, Aegis defers to the Claude # fallback. Mirrors ROUTELET_CONFIDENCE_THRESHOLD in the aegis crate's tuning.rs. -GATE = 0.55 +GATE = 0.95 def score_tfidf(texts: list[str], gold: list[str]) -> float: @@ -174,83 +174,46 @@ def plot_model_comparison(metrics: dict) -> Path: return out -def plot_confidence_histogram(indist: np.ndarray, ood: np.ndarray, gate: float) -> Path: - """Figure 2: routelet confidence on in-distribution holdout vs OOD/garbled - probes, with the gate line. The cascade works if in-distribution input sits - above the gate (kept on-device) while OOD input falls below it (deferred to - Claude). Overlapping (not stacked) histograms, densities so the two groups - are comparable despite different counts.""" - bins = np.linspace(0.0, 1.0, 21) - fig, ax = plt.subplots(figsize=(7, 4.5)) - ax.hist(indist, bins=bins, color=C_INDIST, alpha=0.6, density=True, - label=f"in-distribution holdout (n={len(indist)})", zorder=3) - ax.hist(ood, bins=bins, color=C_OOD, alpha=0.7, density=True, - label=f"OOD / garbled probe (n={len(ood)})", zorder=3) - - ax.axvline(gate, color="black", linestyle="--", linewidth=1.2, zorder=4) - ymax = ax.get_ylim()[1] - ax.text(gate - 0.012, ymax * 0.96, f"{gate:.2f} gate", ha="right", va="top", fontsize=9) - ax.text(gate - 0.012, ymax * 0.55, "← defer to Claude", ha="right", fontsize=8, color="#555") - ax.text(gate + 0.012, ymax * 0.55, "kept on-device →", ha="left", fontsize=8, color="#555") - - ood_deferred = int((ood < gate).sum()) - indist_kept = int((indist >= gate).sum()) - ax.set_title( - f"OOD defers {ood_deferred}/{len(ood)}, in-distribution keeps " - f"{indist_kept}/{len(indist)} at the {gate:.2f} gate", - fontsize=11, fontweight="bold", - ) - ax.set_xlabel("routelet confidence (temperature-scaled max softmax)") - ax.set_ylabel("density") - ax.set_xlim(0, 1) - ax.legend(frameon=False, loc="upper left") - ax.yaxis.grid(True, linestyle="--", alpha=0.4, zorder=0) - ax.set_axisbelow(True) - for spine in ("top", "right"): - ax.spines[spine].set_visible(False) - fig.tight_layout() +def plot_ood_detection( + gate_caught: float, gate_deferred: float, reject_caught: float, reject_deferred: float +) -> Path: + """Figure 2: how each mechanism handles out-of-distribution / garbled input. + "caught" = correctly sent to Claude; "wrongly deferred" = a real command sent + to Claude by mistake. The reject class (a learned "none" label) catches far + more OOD than the confidence gate, at near-zero false-reject cost.""" + methods = ["confidence gate\n(0.95 cutoff)", "reject class\n(learned 'none')"] + caught = [gate_caught * 100, reject_caught * 100] + deferred = [gate_deferred * 100, reject_deferred * 100] - out = OUT_DIR / "confidence_histogram.png" - fig.savefig(out, dpi=150) - plt.close(fig) - return out - - -def plot_deferral_tradeoff(indist: np.ndarray, ood: np.ndarray, gate: float) -> Path: - """Figure 3: as the confidence cutoff rises, what fraction of in-distribution - commands get wrongly deferred to Claude vs what fraction of OOD/garbled input - gets caught. The two lines never separate cleanly, which is why no cutoff - makes the gate work: catching OOD means deferring real commands too.""" - thresholds = np.linspace(0.5, 1.0, 101) - id_deferred = np.array([(indist < t).mean() for t in thresholds]) * 100 - ood_caught = np.array([(ood < t).mean() for t in thresholds]) * 100 - - fig, ax = plt.subplots(figsize=(7.5, 4.5)) - ax.plot(thresholds, ood_caught, color=C_OOD, linewidth=2.4, - label="OOD / garbled caught (good)", zorder=3) - ax.plot(thresholds, id_deferred, color=C_INDIST, linewidth=2.4, - label="real commands wrongly deferred (bad)", zorder=3) - - ax.axvline(gate, color="#777", linestyle="--", linewidth=1.2, zorder=2) - ax.text(gate, 102, f"current cutoff {gate:.2f}", ha="center", va="bottom", - fontsize=8, color="#555") + x = np.arange(len(methods)) + w = 0.36 + fig, ax = plt.subplots(figsize=(7, 4.5)) + b1 = ax.bar(x - w / 2, caught, w, color=C_OOD, + label="OOD / garbled caught (higher better)", zorder=3) + b2 = ax.bar(x + w / 2, deferred, w, color=C_INDIST, + label="real commands wrongly deferred (lower better)", zorder=3) + for bars in (b1, b2): + for bar in bars: + ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 1.5, + f"{bar.get_height():.0f}%", ha="center", va="bottom", + fontsize=10, fontweight="bold") - ax.set_xlabel("confidence cutoff (defer to Claude below it)") - ax.set_ylabel("% of inputs deferred") + ax.set_xticks(x) + ax.set_xticklabels(methods, fontsize=10) + ax.set_ylabel("% of inputs") + ax.set_ylim(0, 100) ax.set_title( - "The 0.55 cutoff is too low: ~0.95 catches far more garbage at little cost", + "Reject class catches far more OOD at near-zero false-reject cost", fontsize=11, fontweight="bold", ) - ax.set_xlim(0.5, 1.0) - ax.set_ylim(0, 105) - ax.legend(loc="upper left", frameon=False) - ax.grid(True, linestyle="--", alpha=0.4, zorder=0) + ax.legend(frameon=False, loc="upper center", fontsize=8) + ax.yaxis.grid(True, linestyle="--", alpha=0.4, zorder=0) ax.set_axisbelow(True) for spine in ("top", "right"): ax.spines[spine].set_visible(False) fig.tight_layout() - out = OUT_DIR / "deferral_tradeoff.png" + out = OUT_DIR / "ood_detection.png" fig.savefig(out, dpi=150) plt.close(fig) return out @@ -272,29 +235,29 @@ def main() -> None: setfit_acc = float(correct.mean()) print(f" SetFit {setfit_acc:.3f}") - # OOD/garbled probe: confidence on inputs the gate is meant to defer. + # OOD detection: compare the old confidence gate to the reject class on the + # same hand-written probe set (caught) and the holdout (wrongly deferred). ood_texts = load_ood_probes() - ood_conf, _ = setfit_predict(bundle, ood_texts) - - # Confidence gate operating point: what the cascade does at GATE. - indist_kept = conf >= GATE - ood_deferred = ood_conf < GATE - gate_stats = { - "threshold": GATE, - "in_distribution_kept_share": round(float(indist_kept.mean()), 3), - "ood_deferred_share": round(float(ood_deferred.mean()), 3), - "kept_accuracy": ( - round(float(correct[indist_kept].mean()), 3) if indist_kept.any() else None - ), + ood_conf, ood_preds = setfit_predict(bundle, ood_texts) + gate_caught = float((ood_conf < GATE).mean()) + reject_caught = float(np.mean([p == "none" for p in ood_preds])) + gate_deferred = float((conf < GATE).mean()) + reject_deferred = float(np.mean([p == "none" for p in preds])) + ood_stats = { + "probe_n": len(ood_texts), + "confidence_gate": {"threshold": GATE, "ood_caught": round(gate_caught, 3), + "real_deferred": round(gate_deferred, 3)}, + "reject_class": {"ood_caught": round(reject_caught, 3), + "real_deferred": round(reject_deferred, 3)}, } - print(f" gate {GATE}: in-dist keeps {gate_stats['in_distribution_kept_share']:.0%}, " - f"OOD defers {gate_stats['ood_deferred_share']:.0%}") + print(f" OOD caught: gate {gate_caught:.0%}, reject-class {reject_caught:.0%}; " + f"real wrongly deferred: gate {gate_deferred:.0%}, reject {reject_deferred:.0%}") metrics: dict = { "eval_n": eval_n, "tfidf": {"accuracy": tfidf_acc}, "setfit": {"accuracy": setfit_acc}, - "gate": gate_stats, + "ood": ood_stats, } haiku = load_haiku(eval_n) if haiku: @@ -307,8 +270,7 @@ def main() -> None: (OUT_DIR / "metrics.json").write_text(json.dumps(metrics, indent=2) + "\n") print(f"wrote {plot_model_comparison(metrics)}") - print(f"wrote {plot_confidence_histogram(conf, ood_conf, GATE)}") - print(f"wrote {plot_deferral_tradeoff(conf, ood_conf, GATE)}") + print(f"wrote {plot_ood_detection(gate_caught, gate_deferred, reject_caught, reject_deferred)}") print(f"wrote {OUT_DIR / 'metrics.json'}") diff --git a/src/routelet/data.py b/src/routelet/data.py index 43538b6..2ffc3d9 100644 --- a/src/routelet/data.py +++ b/src/routelet/data.py @@ -11,7 +11,10 @@ class Intent(StrEnum): - """The label set, frozen. Definitions and boundary rules in docs/taxonomy.md. + """The label set. The first five are the real intents (definitions and + boundary rules in docs/taxonomy.md). NONE is the reject class: out-of- + distribution or garbled input the router should not act on, used to give the + model an explicit "I don't know" instead of confidently mislabeling junk. StrEnum gives two things we rely on: members serialize as plain strings, and Intent("typo") raises ValueError, which is how labels get validated on load. @@ -22,6 +25,7 @@ class Intent(StrEnum): CHAT = "chat" MEMORY = "memory" AGENT = "agent" + NONE = "none" @dataclass(frozen=True, slots=True) diff --git a/src/routelet/teacher.py b/src/routelet/teacher.py index 5d17211..b37e0dd 100644 --- a/src/routelet/teacher.py +++ b/src/routelet/teacher.py @@ -44,9 +44,13 @@ If a command still fits more than one after these rules, pick the first match in this order: agent, memory, integration, find_action, chat.""" +# Real intents only: the teacher labels actual commands, never the reject class +# (NONE), which is learned from generated OOD data, not from the LLM. +_REAL_INTENTS = [i.value for i in Intent if i is not Intent.NONE] + INTENT_SCHEMA = { "type": "object", - "properties": {"intent": {"type": "string", "enum": [i.value for i in Intent]}}, + "properties": {"intent": {"type": "string", "enum": _REAL_INTENTS}}, "required": ["intent"], "additionalProperties": False, }