Skip to content

Commit 80cb9fa

Browse files
author
Your Name
committed
Update benchmark suite to be able to more easily run comparison batches
1 parent c72fca0 commit 80cb9fa

5 files changed

Lines changed: 653 additions & 9 deletions

File tree

‎benchmark/benchmark.py‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -515,7 +515,8 @@ def load_results(results_dir, stats_languages=None):
515515
results["test_cases_total"] = total_cases
516516
if passed_cases is not None:
517517
results["test_cases_passed"] = passed_cases
518-
518+
if passed_cases is not None and total_cases is not None:
519+
results["test_cases_pass_ratio"] = passed_cases / total_cases
519520
# Update the JSON file immediately to keep data fresh
520521
fname.write_text(json.dumps(results, indent=4))
521522
logger.debug(f"Updated {fname} with test case counts")
@@ -583,7 +584,8 @@ def summarize_results(results_dir, verbose, stats_languages=None):
583584
res.completion_tokens = 0
584585
res.test_cases_total = 0
585586
res.test_cases_passed = 0
586-
587+
res.test_cases_pass_ratio_sum = 0.0
588+
res.test_cases_pass_ratio_count = 0
587589
res.reasoning_effort = None
588590
res.thinking_tokens = None
589591
res.map_tokens = None
@@ -621,6 +623,8 @@ def add(attr_name, increment, global_stats, lang_stats):
621623
lang_stats.completion_tokens = 0
622624
lang_stats.test_cases_total = 0
623625
lang_stats.test_cases_passed = 0
626+
lang_stats.test_cases_pass_ratio_sum = 0.0
627+
lang_stats.test_cases_pass_ratio_count = 0
624628
lang_to_stats[lang] = lang_stats
625629
lang_to_passed_tests[lang] = [0] * tries
626630

@@ -682,6 +686,11 @@ def add(attr_name, increment, global_stats, lang_stats):
682686
if passed_cases is not None:
683687
add("test_cases_passed", passed_cases, res, lang_stats)
684688

689+
pass_ratio = results.get("test_cases_pass_ratio")
690+
if pass_ratio is not None:
691+
add("test_cases_pass_ratio_sum", pass_ratio, res, lang_stats)
692+
add("test_cases_pass_ratio_count", 1, res, lang_stats)
693+
685694
res.reasoning_effort = results.get("reasoning_effort")
686695
res.thinking_tokens = results.get("thinking_tokens")
687696
res.map_tokens = results.get("map_tokens")
@@ -764,6 +773,12 @@ def show(stat, red="red"):
764773
res.test_cases_percentage = 100 * res.test_cases_passed / res.test_cases_total
765774
print(f" test_cases_percentage: {res.test_cases_percentage:.1f}")
766775

776+
if res.test_cases_pass_ratio_count > 0:
777+
res.test_cases_pass_ratio_avg = (
778+
100 * res.test_cases_pass_ratio_sum / res.test_cases_pass_ratio_count
779+
)
780+
print(f" test_cases_pass_ratio_avg: {res.test_cases_pass_ratio_avg:.1f}")
781+
767782
if variants["model"]:
768783
a_model = set(variants["model"]).pop()
769784
command = f"cecli --model {a_model}"
@@ -801,6 +816,18 @@ def format_lang_stats(lang, lang_stats):
801816
pass_rate = 100 * num_passed / float(lang_stats.completed_tests)
802817
setattr(lang_stats, f"pass_rate_{i + 1}", pass_rate)
803818

819+
if lang_stats.test_cases_pass_ratio_count > 0:
820+
lang_stats.test_cases_pass_ratio_avg = (
821+
100
822+
* lang_stats.test_cases_pass_ratio_sum
823+
/ lang_stats.test_cases_pass_ratio_count
824+
)
825+
else:
826+
lang_stats.test_cases_pass_ratio_avg = 0.0
827+
828+
del lang_stats.test_cases_pass_ratio_sum
829+
del lang_stats.test_cases_pass_ratio_count
830+
804831
# Then format attributes into ready-to-print strings
805832
for attr in lang_stats.__dict__:
806833
val = getattr(lang_stats, attr)
@@ -870,6 +897,11 @@ def compute_lang_to_col_widths(lang_to_stats):
870897
console.rule()
871898

872899
# print(json.dumps(vars(res), indent=4, sort_keys=True))
900+
if hasattr(res, "test_cases_pass_ratio_sum"):
901+
del res.test_cases_pass_ratio_sum
902+
if hasattr(res, "test_cases_pass_ratio_count"):
903+
del res.test_cases_pass_ratio_count
904+
873905
return res
874906

875907

@@ -1156,6 +1188,7 @@ async def run_test_real(
11561188
verbose=verbose,
11571189
yes_always_commands=True,
11581190
max_reflections=0,
1191+
file_diffs=False,
11591192
),
11601193
map_mul_no_files=4,
11611194
mcp_manager=None,

‎benchmark/variations.1.sh‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
#!/bin/bash
2+
# Benchmark runner script for testing multiple OpenRouter models
3+
# Usage: ./run_benchmark_variations.sh [OPTIONS]
4+
5+
set -e # Exit on error
6+
7+
# Default values
8+
BASE_NAME="cecli-little-guys-h6"
9+
EDIT_FORMAT="hashline"
10+
MAP_TOKENS="512"
11+
THREADS="1"
12+
LANGUAGES="javascript,python,rust,go,java"
13+
HASH_RE="^.[15ef]"
14+
NUM_TESTS="72"
15+
EXERCISES_DIR="polyglot-benchmark"
16+
OUTPUT_DIR="tmp.benchmarks"
17+
SLEEP_BETWEEN=30 # Seconds to sleep between runs
18+
19+
# List of models to test
20+
# RERUN
21+
# "openrouter/minimax/minimax-m2.1"
22+
# "openrouter/qwen/qwen3-vl-235b-a22b-thinking"
23+
MODELS=(
24+
# "openrouter/qwen/qwen3.5-35b-a3b"
25+
# "openrouter/xiaomi/mimo-v2-flash"
26+
# "openrouter/moonshotai/kimi-k2.5"
27+
# "openrouter/minimax/minimax-m2.5"
28+
"openrouter/anthropic/claude-haiku-4.5"
29+
"openrouter/openai/gpt-oss-120b"
30+
"openrouter/openai/gpt-5-mini"
31+
"openrouter/google/gemini-3-flash-preview"
32+
"openrouter/deepseek/deepseek-v3.2-exp"
33+
)
34+
35+
# Parse command line arguments
36+
while [[ $# -gt 0 ]]; do
37+
case $1 in
38+
--base-name)
39+
BASE_NAME="$2"
40+
shift 2
41+
;;
42+
--edit-format)
43+
EDIT_FORMAT="$2"
44+
shift 2
45+
;;
46+
--map-tokens)
47+
MAP_TOKENS="$2"
48+
shift 2
49+
;;
50+
--threads)
51+
THREADS="$2"
52+
shift 2
53+
;;
54+
--hash-re)
55+
HASH_RE="$2"
56+
shift 2
57+
;;
58+
--num-tests)
59+
NUM_TESTS="$2"
60+
shift 2
61+
;;
62+
--exercises-dir)
63+
EXERCISES_DIR="$2"
64+
shift 2
65+
;;
66+
--output-dir)
67+
OUTPUT_DIR="$2"
68+
shift 2
69+
;;
70+
--sleep)
71+
SLEEP_BETWEEN="$2"
72+
shift 2
73+
;;
74+
--help)
75+
echo "Usage: $0 [OPTIONS]"
76+
echo ""
77+
echo "Options:"
78+
echo " --base-name NAME Base name for benchmark runs (default: $BASE_NAME)"
79+
echo " --edit-format FORMAT Edit format to use (default: $EDIT_FORMAT)"
80+
echo " --map-tokens TOKENS Map tokens (default: $MAP_TOKENS)"
81+
echo " --threads N Number of threads (default: $THREADS)"
82+
echo " --hash-re REGEX Hash regex filter (default: $HASH_RE)"
83+
echo " --num-tests N Number of tests to run (default: $NUM_TESTS)"
84+
echo " --exercises-dir DIR Exercises directory (default: $EXERCISES_DIR)"
85+
echo " --output-dir DIR Output directory (default: $OUTPUT_DIR)"
86+
echo " --sleep SECONDS Sleep between runs in seconds (default: $SLEEP_BETWEEN)"
87+
echo " --help Show this help message"
88+
echo ""
89+
echo "Example:"
90+
echo " $0 --threads 2 --num-tests 5"
91+
exit 0
92+
;;
93+
*)
94+
echo "Unknown option: $1"
95+
echo "Use --help for usage information"
96+
exit 1
97+
;;
98+
esac
99+
done
100+
101+
# Function to run a single benchmark
102+
run_benchmark() {
103+
local model="$1"
104+
local run_name="$2"
105+
106+
echo "========================================================================"
107+
echo "Starting benchmark: $run_name"
108+
echo "Model: $model"
109+
echo "Time: $(date)"
110+
echo "========================================================================"
111+
112+
# Create the benchmark command
113+
./benchmark/benchmark.py "$run_name" \
114+
--new \
115+
--model "$model" \
116+
--edit-format "$EDIT_FORMAT" \
117+
--map-tokens "$MAP_TOKENS" \
118+
--threads "$THREADS" \
119+
--hash-re "$HASH_RE" \
120+
--num-tests "$NUM_TESTS" \
121+
--languages "$LANGUAGES" \
122+
--tries 2 \
123+
--exercises-dir "$EXERCISES_DIR"
124+
125+
echo "Benchmark completed: $run_name"
126+
echo "Results directory: $OUTPUT_DIR/$(ls -t $OUTPUT_DIR | grep "$run_name" | head -1)"
127+
echo ""
128+
}
129+
130+
# Function to generate statistics for all completed runs
131+
generate_stats() {
132+
echo "========================================================================"
133+
echo "Generating statistics for all completed runs"
134+
echo "========================================================================"
135+
136+
for dir in "$OUTPUT_DIR"/*; do
137+
if [ -d "$dir" ] && [ -f "$dir/.cecli.results.json" ]; then
138+
echo "Processing: $(basename "$dir")"
139+
./benchmark/benchmark.py --stats "$dir" || true
140+
echo ""
141+
fi
142+
done
143+
}
144+
145+
# Main execution
146+
main() {
147+
echo "========================================================================"
148+
echo "OpenRouter Model Benchmark Runner"
149+
echo "========================================================================"
150+
echo "Configuration:"
151+
echo " Base name: $BASE_NAME"
152+
echo " Edit format: $EDIT_FORMAT"
153+
echo " Map tokens: $MAP_TOKENS"
154+
echo " Threads: $THREADS"
155+
echo " Hash regex: $HASH_RE"
156+
echo " Num tests: $NUM_TESTS"
157+
echo " Exercises dir: $EXERCISES_DIR"
158+
echo " Output dir: $OUTPUT_DIR"
159+
echo " Sleep between: ${SLEEP_BETWEEN}s"
160+
echo " Models to test: ${#MODELS[@]}"
161+
echo ""
162+
163+
# Create output directory if it doesn't exist
164+
mkdir -p "$OUTPUT_DIR"
165+
166+
# Run benchmarks for each model
167+
for model in "${MODELS[@]}"; do
168+
# Create a run name by replacing slashes with hyphens
169+
local model_slug=$(echo "$model" | sed 's|/|-|g')
170+
local run_name="${BASE_NAME}-${model_slug}"
171+
172+
run_benchmark "$model" "$run_name"
173+
174+
# Sleep between runs to avoid rate limiting
175+
if [ "$SLEEP_BETWEEN" -gt 0 ]; then
176+
echo "Sleeping for ${SLEEP_BETWEEN} seconds before next run..."
177+
sleep "$SLEEP_BETWEEN"
178+
echo ""
179+
fi
180+
done
181+
182+
# Generate statistics
183+
generate_stats
184+
185+
echo "========================================================================"
186+
echo "All benchmarks completed!"
187+
echo "========================================================================"
188+
echo ""
189+
echo "Summary of results directories:"
190+
ls -la "$OUTPUT_DIR" | grep "$BASE_NAME"
191+
echo ""
192+
echo "To view statistics for a specific run:"
193+
echo " ./benchmark/benchmark.py --stats $OUTPUT_DIR/<run-directory>"
194+
echo ""
195+
echo "To compare all results:"
196+
echo " for dir in $OUTPUT_DIR/*$BASE_NAME*; do"
197+
echo " echo \"=== \$(basename \$dir) ===\""
198+
echo " ./benchmark/benchmark.py --stats \"\$dir\" 2>/dev/null | grep -E '(pass_rate|total_cost|completed_tests)' || true"
199+
echo " done"
200+
}
201+
202+
# Run main function
203+
main

0 commit comments

Comments
 (0)