From ea61bbbe08e01982f6adfba73af8f9c6a6881898 Mon Sep 17 00:00:00 2001 From: daniel-lxs Date: Thu, 6 Aug 2026 11:33:06 -0500 Subject: [PATCH] data: publish explorer results --- evals.json | 174 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 162 insertions(+), 12 deletions(-) diff --git a/evals.json b/evals.json index 4a88069..aa14c6f 100644 --- a/evals.json +++ b/evals.json @@ -1,6 +1,6 @@ { "format": "roomote-role-evals/v1", - "publishedAt": "2026-08-06T16:19:43.537Z", + "publishedAt": "2026-08-06T16:33:05.152Z", "evals": [ { "id": "coding", @@ -3093,29 +3093,29 @@ }, { "model": "google/gemini-3.6-flash", - "qualityScore": 41, - "latencyP50Ms": 16540, - "latencyP95Ms": 50735, - "costUsd": 0.0643035, - "costPerCaseUsd": 0.0049464230769230766, - "tokens": 42869, + "qualityScore": 39, + "latencyP50Ms": 1499, + "latencyP95Ms": 2190, + "costUsd": 0.0057165, + "costPerCaseUsd": 0.00043973076923076924, + "tokens": 3811, "casesAttempted": 13, "metrics": [ { "label": "file recall", - "value": "47%", + "value": "53%", "hint": "of the files the real fix touched, how many were predicted — higher is better", - "sort": 0.47435897435897434 + "sort": 0.5256410256410257 }, { "label": "precision", - "value": "36%", + "value": "31%", "hint": "of predicted files, how many were right", - "sort": 0.3589743589743589 + "sort": 0.3076923076923077 }, { "label": "latency p50/p95", - "value": "16.5s / 50.7s", + "value": "1.5s / 2.2s", "hint": "wall-clock per attempted file-targeting call; failures count as the 60s timeout", "sort": null } @@ -3181,6 +3181,96 @@ } ] }, + { + "model": "openai/gpt-5.4-mini", + "qualityScore": 33, + "latencyP50Ms": 2652, + "latencyP95Ms": 11390, + "costUsd": 0.0026115, + "costPerCaseUsd": 0.0002008846153846154, + "tokens": 3482, + "casesAttempted": 13, + "metrics": [ + { + "label": "file recall", + "value": "37%", + "hint": "of the files the real fix touched, how many were predicted — higher is better", + "sort": 0.37179487179487175 + }, + { + "label": "precision", + "value": "29%", + "hint": "of predicted files, how many were right", + "sort": 0.2948717948717948 + }, + { + "label": "latency p50/p95", + "value": "2.7s / 11.4s", + "hint": "wall-clock per attempted file-targeting call; failures count as the 60s timeout", + "sort": null + } + ] + }, + { + "model": "qwen/qwen3.7-plus", + "qualityScore": 33, + "latencyP50Ms": 4832, + "latencyP95Ms": 60000, + "costUsd": 0.00206752, + "costPerCaseUsd": 0.00015904000000000002, + "tokens": 6461, + "casesAttempted": 13, + "metrics": [ + { + "label": "file recall", + "value": "41%", + "hint": "of the files the real fix touched, how many were predicted — higher is better", + "sort": 0.4102564102564103 + }, + { + "label": "precision", + "value": "28%", + "hint": "of predicted files, how many were right", + "sort": 0.28205128205128205 + }, + { + "label": "latency p50/p95", + "value": "4.8s / 60.0s", + "hint": "wall-clock per attempted file-targeting call; failures count as the 60s timeout", + "sort": null + } + ] + }, + { + "model": "google/gemini-3.5-flash", + "qualityScore": 33, + "latencyP50Ms": 12000, + "latencyP95Ms": 60000, + "costUsd": 0.023796, + "costPerCaseUsd": 0.0018304615384615386, + "tokens": 15864, + "casesAttempted": 13, + "metrics": [ + { + "label": "file recall", + "value": "39%", + "hint": "of the files the real fix touched, how many were predicted — higher is better", + "sort": 0.3910256410256411 + }, + { + "label": "precision", + "value": "28%", + "hint": "of predicted files, how many were right", + "sort": 0.27884615384615385 + }, + { + "label": "latency p50/p95", + "value": "12.0s / 60.0s", + "hint": "wall-clock per attempted file-targeting call; failures count as the 60s timeout", + "sort": null + } + ] + }, { "model": "moonshotai/kimi-k3", "qualityScore": 33, @@ -3421,6 +3511,66 @@ } ] }, + { + "model": "google/gemini-3.5-flash-lite", + "qualityScore": 24, + "latencyP50Ms": 1333, + "latencyP95Ms": 2095, + "costUsd": 0.0010578, + "costPerCaseUsd": 0.00008136923076923076, + "tokens": 3526, + "casesAttempted": 13, + "metrics": [ + { + "label": "file recall", + "value": "33%", + "hint": "of the files the real fix touched, how many were predicted — higher is better", + "sort": 0.3333333333333333 + }, + { + "label": "precision", + "value": "19%", + "hint": "of predicted files, how many were right", + "sort": 0.19230769230769232 + }, + { + "label": "latency p50/p95", + "value": "1.3s / 2.1s", + "hint": "wall-clock per attempted file-targeting call; failures count as the 60s timeout", + "sort": null + } + ] + }, + { + "model": "mistralai/ministral-14b-2512", + "qualityScore": 17, + "latencyP50Ms": 1709, + "latencyP95Ms": 60000, + "costUsd": 0.0007109999999999999, + "costPerCaseUsd": 0.000054692307692307686, + "tokens": 3555, + "casesAttempted": 13, + "metrics": [ + { + "label": "file recall", + "value": "29%", + "hint": "of the files the real fix touched, how many were predicted — higher is better", + "sort": 0.2948717948717948 + }, + { + "label": "precision", + "value": "12%", + "hint": "of predicted files, how many were right", + "sort": 0.1163003663003663 + }, + { + "label": "latency p50/p95", + "value": "1.7s / 60.0s", + "hint": "wall-clock per attempted file-targeting call; failures count as the 60s timeout", + "sort": null + } + ] + }, { "model": "deepseek/deepseek-v4-flash-0731", "qualityScore": 15,