From 83526d50f862f75febc48b57e2411d33741367d0 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:09:52 -0700 Subject: [PATCH 01/16] Adding support for multiple-choice custom_nocode benchmarks --- CONFIG.md | 29 +- README.md | 2 +- cli/README.md | 1 - cli/src/builtins/common.rs | 51 ++- cli/src/builtins/custom_nocode.rs | 432 +++++++++++++++++--- cli/src/builtins/dataset_runner.rs | 3 +- cli/src/commands/resume.rs | 37 +- cli/src/commands/run.rs | 75 +++- cli/src/config/custom_nocode.rs | 163 +++++++- cli/src/config/mod.rs | 5 +- cli/src/dataset/cache.rs | 60 ++- cli/src/dataset/mod.rs | 2 + custom-nocode-examples/prompts/gpqa.txt | 5 + custom-nocode-examples/prompts/medmcqa.txt | 5 + custom-nocode-examples/prompts/medqa.txt | 5 + custom-nocode-examples/prompts/mmlu-pro.txt | 5 + custom-nocode-examples/prompts/qa.txt | 2 +- custom-nocode-examples/quantiles.toml | 56 ++- 18 files changed, 804 insertions(+), 134 deletions(-) create mode 100644 custom-nocode-examples/prompts/gpqa.txt create mode 100644 custom-nocode-examples/prompts/medmcqa.txt create mode 100644 custom-nocode-examples/prompts/medqa.txt create mode 100644 custom-nocode-examples/prompts/mmlu-pro.txt diff --git a/CONFIG.md b/CONFIG.md index a5968f6..b851941 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -109,7 +109,6 @@ style = "qa" dataset = "quantiles/simpleqa-verified" model = "random" prompt_template_file = "prompts/qa.txt" -prompt_column = "problem" golden_column = "answer" limit = 10 ``` @@ -127,13 +126,35 @@ For a complete minimal example, see [`custom-nocode-examples/quantiles.toml`](./ | `type` | string | yes | Must be `"custom_nocode"`. | | `style` | string | yes | Must be `"qa"`. | | `dataset` | string | yes | Dataset identifier, for example `"quantiles/simpleqa-verified"`. | +| `dataset_config` | string | no | Hugging Face dataset configuration or subset. | +| `split` | string | no | Dataset split. When omitted, Quantiles selects a standard evaluation split. | +| `revision` | string | no | Dataset revision. | | `model` | string or table | no | Model sampler. Defaults to the demo random sampler. See [model naming](#model-naming). | -| `prompt_template_file` | string | yes | Path to a Jinja prompt template file. The template receives `prompt` from the configured prompt column. | -| `prompt_column` | string | yes | Dataset column containing the prompt text. | -| `golden_column` | string | yes | Dataset column containing the golden answer. | +| `prompt_template_file` | string | yes | Path to a Jinja prompt template file. The template receives the complete dataset `row` and, for multiple-choice benchmarks, normalized `choices`. | +| `golden_column` | string | conditional | Dataset column containing the golden answer text or label. Exactly one golden answer source is required. | +| `golden_index_column` | string | conditional | Dataset column containing the golden choice index. | +| `golden_index_base` | integer | no | Index base for `golden_index_column`. Defaults to `0`. | +| `correct_choice_column` | string | conditional | Choice column known to contain the correct answer, useful when choices are shuffled. | +| `choices_column` | string | no | Dataset column containing choices as an array or label-keyed object. | +| `choice_columns` | array of strings | no | Dataset columns containing choices in their original order. | +| `choice_labels` | array of strings | conditional | Labels assigned to choices in order. Required for multiple choice; array-backed rows may use a prefix of the configured labels. | +| `shuffle_choices` | boolean | no | Deterministically shuffle choices. Defaults to `false`. | +| `shuffle_seed_column` | string | conditional | Stable row identifier used when `shuffle_choices = true`. | | `limit` | integer | no | Number of dataset rows to evaluate. | | `max_workers` | integer | no | Maximum concurrent workers. | +Templates access dataset fields directly. A multiple-choice template can iterate the normalized choices: + +```jinja +{{ row.question }} + +{% for choice in choices %} +{{ choice.label }}. {{ choice.text }} +{% endfor %} +``` + +See [`custom-nocode-examples/quantiles.toml`](./custom-nocode-examples/quantiles.toml) for runnable MedQA, MedMCQA, MMLU-Pro, and GPQA configurations. + ### `custom_code` Custom evaluations are external programs built with the Quantiles Python SDK. Their config sections contain the following fields: diff --git a/README.md b/README.md index e67fab4..45f8c43 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ Both the CLI and Python SDK support offline benchmark workflows, including the f Built-in benchmarks are ready-to-run evaluations with predefined datasets, scoring methodologies, and metrics. Use them when you want a standardized evaluation that provides a common reference point, a repeatable baseline, or a well-defined implementation of an industry benchmark. -For dataset-backed QA checks that do not need custom Python or TypeScript code, use a `custom_nocode` benchmark in `quantiles.toml`. A `custom_nocode` QA benchmark points to a dataset, a Jinja prompt template, the dataset column containing the prompt, and the dataset column containing the golden answer. See [`custom-nocode-examples/quantiles.toml`](./custom-nocode-examples/quantiles.toml) for a minimal example. +For dataset-backed QA checks that do not need custom Python or TypeScript code, use a `custom_nocode` benchmark in `quantiles.toml`. A `custom_nocode` QA benchmark points to a dataset, renders a Jinja prompt from the complete dataset row, and supports exact-answer and multiple-choice scoring. See [`custom-nocode-examples/quantiles.toml`](./custom-nocode-examples/quantiles.toml) for runnable SimpleQA Verified, MedQA, MedMCQA, MMLU-Pro, and GPQA configurations. | Code | When to use | | --- | --- | diff --git a/cli/README.md b/cli/README.md index 2ce5c51..68578e1 100644 --- a/cli/README.md +++ b/cli/README.md @@ -82,7 +82,6 @@ style = "qa" dataset = "quantiles/simpleqa-verified" model = "random" prompt_template_file = "prompts/qa.txt" -prompt_column = "problem" golden_column = "answer" limit = 10 ``` diff --git a/cli/src/builtins/common.rs b/cli/src/builtins/common.rs index 8b119c1..8b15483 100644 --- a/cli/src/builtins/common.rs +++ b/cli/src/builtins/common.rs @@ -326,44 +326,51 @@ mod tests { } #[rstest] - fn test_resolve_sampler_uses_default_when_none() { - let result = resolve_sampler(None, || Arc::new(crate::llm::random::RandomSampler::new(80))) - .unwrap(); + #[tokio::test] + async fn test_resolve_sampler_uses_default_when_none() { + let result = resolve_sampler(None, || { + Arc::new(crate::llm::random::RandomSampler::new(80)) + }) + .unwrap(); // Should get the default sampler, not panic or error. - assert!(!result.sample("test").block_on().is_empty()); + assert!(!result.sample("test").await.unwrap().is_empty()); } #[rstest] - fn test_resolve_sampler_resolves_configured_sampler() { + #[tokio::test] + async fn test_resolve_sampler_resolves_configured_sampler() { let sampler = crate::llm::Sampler::Random {}; let result = resolve_sampler(Some(&sampler), || { panic!("default should not be called when model is Some") }) .unwrap(); // Should get a resolved sampler, not the default. - assert!(!result.sample("test").block_on().is_empty()); + assert!(!result.sample("test").await.unwrap().is_empty()); } #[rstest] fn test_emit_accuracy_metrics_empty() { let tmpdir = tempfile::tempdir().unwrap(); - let metrics_store = crate::metrics_store::MetricsStore::new(tmpdir.path().to_path_buf()) - .unwrap(); + let metrics_store = + crate::metrics_store::MetricsStore::new(tmpdir.path().to_path_buf()).unwrap(); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async { emit_accuracy_metrics(&metrics_store, 1, [false; 0]).await; metrics_store.flush(1).await.unwrap(); let agg = metrics_store.list_aggregate_for_run(1).await.unwrap(); - assert!(agg.is_empty(), "no metrics should be emitted for empty results"); + assert!( + agg.is_empty(), + "no metrics should be emitted for empty results" + ); }); } #[rstest] fn test_emit_accuracy_metrics_all_correct() { let tmpdir = tempfile::tempdir().unwrap(); - let metrics_store = crate::metrics_store::MetricsStore::new(tmpdir.path().to_path_buf()) - .unwrap(); + let metrics_store = + crate::metrics_store::MetricsStore::new(tmpdir.path().to_path_buf()).unwrap(); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async { @@ -374,19 +381,22 @@ mod tests { let accuracy = agg.iter().find(|m| m.metric_name == "accuracy").unwrap(); assert!((accuracy.metric_value - 1.0).abs() < 1e-10); - let correct = agg.iter().find(|m| m.metric_name == "correct_count").unwrap(); - assert_eq!(correct.metric_value as i64, 3); + let correct = agg + .iter() + .find(|m| m.metric_name == "correct_count") + .unwrap(); + assert!((correct.metric_value - 3.0).abs() < f64::EPSILON); let total = agg.iter().find(|m| m.metric_name == "total_count").unwrap(); - assert_eq!(total.metric_value as i64, 3); + assert!((total.metric_value - 3.0).abs() < f64::EPSILON); }); } #[rstest] fn test_emit_accuracy_metrics_mixed() { let tmpdir = tempfile::tempdir().unwrap(); - let metrics_store = crate::metrics_store::MetricsStore::new(tmpdir.path().to_path_buf()) - .unwrap(); + let metrics_store = + crate::metrics_store::MetricsStore::new(tmpdir.path().to_path_buf()).unwrap(); let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async { @@ -397,11 +407,14 @@ mod tests { let accuracy = agg.iter().find(|m| m.metric_name == "accuracy").unwrap(); assert!((accuracy.metric_value - 0.5).abs() < 1e-10); - let correct = agg.iter().find(|m| m.metric_name == "correct_count").unwrap(); - assert_eq!(correct.metric_value as i64, 2); + let correct = agg + .iter() + .find(|m| m.metric_name == "correct_count") + .unwrap(); + assert!((correct.metric_value - 2.0).abs() < f64::EPSILON); let total = agg.iter().find(|m| m.metric_name == "total_count").unwrap(); - assert_eq!(total.metric_value as i64, 4); + assert!((total.metric_value - 4.0).abs() < f64::EPSILON); }); } } diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index 3f1bf33..a69cc03 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -4,8 +4,7 @@ use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; use crate::builtins::common::{ - emit_accuracy_metrics, extract_text, get_max_workers, hash_input, resolve_sampler, - run_timed_step, + emit_accuracy_metrics, get_max_workers, hash_input, resolve_sampler, run_timed_step, }; use crate::builtins::dataset_runner::DatasetRunner; use crate::builtins::input::set_builtin_run_input; @@ -19,6 +18,9 @@ use crate::llm::random::RandomSampler; struct CustomNoCodeInput { style: crate::config::CustomNoCodeStyle, dataset: String, + dataset_config: Option, + split: Option, + revision: Option, #[serde(default)] model: Option, #[serde(flatten)] @@ -30,20 +32,31 @@ struct CustomNoCodeInput { struct RowOutput { input: String, response: String, + parsed_response: Option, golden: String, is_correct: bool, } +#[derive(Clone, Debug, Serialize)] +struct PromptChoice { + label: String, + text: String, +} + +struct PreparedRow { + choices: Vec, + golden: String, + is_multiple_choice: bool, +} + /// Arguments for the [`CustomNoCodeBuiltin::evaluate_row`] method struct EvaluateRowArgs<'a> { /// The row index i: usize, /// The row value row: &'a serde_json::Value, - /// The name of the column in this row that has the prompt - prompt_column: &'a str, - /// The name of the column in this row that has the golden answer - golden_column: &'a str, + /// QA and multiple-choice configuration. + qa: &'a crate::config::CustomNoCodeQaConfig, /// The prompt template template_str: &'a str, /// The pre-constructed jinja template env @@ -73,26 +86,19 @@ impl CustomNoCodeBuiltin { } async fn evaluate_row(&self, args: EvaluateRowArgs<'_>) -> Result { - let prompt = extract_text(args.row, args.prompt_column).with_context(|| { - format!( - "row {}: missing prompt column `{}`", - args.i, args.golden_column - ) - })?; - let golden = extract_text(args.row, args.golden_column).with_context(|| { - format!( - "row {}: missing golden column `{}`", - args.i, args.golden_column - ) - })?; + let prepared = prepare_row(args.i, args.row, args.qa)?; let rendered = args .env - .render_str(args.template_str, jinja::context!(prompt => &prompt)) + .render_str( + args.template_str, + jinja::context!(row => args.row, choices => &prepared.choices), + ) .with_context(|| format!("row {}: failed to render prompt template", args.i))?; let input_hash = hash_input(&format!( - "{rendered}\nmodel={}\nworkflow={}", + "{rendered}\ngolden={}\nmodel={}\nworkflow={}", + prepared.golden, args.model_name, self.name() )); @@ -111,12 +117,21 @@ impl CustomNoCodeBuiltin { .await .with_context(|| format!("failed to sample LLM for row {}", args.i))?; - let is_correct = is_exact_match(&model_response, &golden); + let parsed_response = if prepared.is_multiple_choice { + extract_choice_label( + &model_response, + args.qa.choice_labels.as_deref().unwrap_or_default(), + ) + } else { + Some(model_response.trim().to_owned()) + }; + let is_correct = parsed_response.as_deref() == Some(prepared.golden.trim()); Ok(RowOutput { input: rendered.clone(), response: model_response, - golden, + parsed_response, + golden: prepared.golden, is_correct, }) }, @@ -151,8 +166,14 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { let max_workers = config.qa.max_workers.unwrap_or_else(get_max_workers); let llm = resolve_sampler(config.model.as_ref(), || Arc::new(RandomSampler::new(80)))?; - let (manager, info, limit) = - resolve_dataset_limit(&config.dataset, config.qa.limit).await?; + let (manager, info, limit) = resolve_dataset_limit( + &config.dataset, + config.dataset_config.as_deref(), + config.split.as_deref(), + config.revision.as_deref(), + config.qa.limit, + ) + .await?; set_builtin_run_input( ctx.db, ctx.run_id, @@ -169,8 +190,7 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { .map_or("random".to_string(), std::string::ToString::to_string); let run_id = ctx.run_id; let dataset = config.dataset.clone(); - let prompt_column = config.qa.prompt_column.clone(); - let golden_column = config.qa.golden_column.clone(); + let qa = Arc::new(config.qa.clone()); let template_str = Arc::new(template_str); let name = self.name(); @@ -182,15 +202,13 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { let db = db.clone(); let model_name = model_name.clone(); let template_str = Arc::clone(&template_str); - let prompt_column = prompt_column.clone(); - let golden_column = golden_column.clone(); + let qa = Arc::clone(&qa); let env = env.clone(); async move { let args = EvaluateRowArgs { i, row: &row, - prompt_column: &prompt_column, - golden_column: &golden_column, + qa: &qa, template_str: &template_str, env: &env, model_name: &model_name, @@ -235,17 +253,25 @@ fn load_template(path: &str) -> Result<(String, jinja::Environment<'_>)> { let template_str = std::fs::read_to_string(path) .with_context(|| format!("failed to read prompt template file `{path}`"))?; let env = jinja::Environment::new(); - env.render_str(&template_str, jinja::context!(prompt => "")) - .with_context(|| format!("invalid jinja syntax in prompt template file `{path}`"))?; + env.render_str( + &template_str, + jinja::context!(row => serde_json::json!({}), choices => Vec::::new()), + ) + .with_context(|| format!("invalid jinja syntax in prompt template file `{path}`"))?; Ok((template_str, env)) } async fn resolve_dataset_limit( dataset: &str, + dataset_config: Option<&str>, + split: Option<&str>, + revision: Option<&str>, limit: Option, ) -> Result<(DatasetManager, crate::dataset::DatasetInfo, usize)> { let manager = DatasetManager::new()?; - let info = manager.init(dataset, None, None, None).await?; + let info = manager + .init(dataset, dataset_config, split, revision) + .await?; let total = info .total_rows @@ -255,9 +281,219 @@ async fn resolve_dataset_limit( Ok((manager, info, limit)) } -/// Case-sensitive exact-match comparison after trimming whitespace. -fn is_exact_match(response: &str, golden: &str) -> bool { - response.trim() == golden.trim() +fn prepare_row( + row_index: usize, + row: &serde_json::Value, + config: &crate::config::CustomNoCodeQaConfig, +) -> Result { + let Some(labels) = config.choice_labels.as_ref() else { + let column = config + .golden_column + .as_deref() + .context("validated custom_nocode config has no golden column")?; + let golden = extract_scalar(row, column) + .with_context(|| format!("row {row_index}: missing golden column `{column}`"))?; + return Ok(PreparedRow { + choices: Vec::new(), + golden, + is_multiple_choice: false, + }); + }; + + let choice_values = extract_choices(row_index, row, config, labels)?; + if choice_values.is_empty() || choice_values.len() > labels.len() { + bail!( + "row {row_index}: found {} choices but configured only {} labels", + choice_values.len(), + labels.len() + ); + } + let labels = &labels[..choice_values.len()]; + + let correct_index = resolve_correct_index(row_index, row, config, labels, &choice_values)?; + let mut indexed_choices: Vec<(String, bool)> = choice_values + .into_iter() + .enumerate() + .map(|(index, text)| (text, index == correct_index)) + .collect(); + + if config.shuffle_choices { + let seed_column = config + .shuffle_seed_column + .as_deref() + .context("validated shuffle config has no seed column")?; + let seed = extract_scalar(row, seed_column).with_context(|| { + format!("row {row_index}: missing shuffle seed column `{seed_column}`") + })?; + deterministic_shuffle(&mut indexed_choices, &seed); + } + + let mut golden = None; + let choices = indexed_choices + .into_iter() + .zip(labels) + .map(|((text, is_correct), label)| { + if is_correct { + golden = Some(label.clone()); + } + PromptChoice { + label: label.clone(), + text, + } + }) + .collect(); + + Ok(PreparedRow { + choices, + golden: golden.context("correct choice disappeared while assigning labels")?, + is_multiple_choice: true, + }) +} + +fn extract_choices( + row_index: usize, + row: &serde_json::Value, + config: &crate::config::CustomNoCodeQaConfig, + labels: &[String], +) -> Result> { + if let Some(columns) = &config.choice_columns { + return columns + .iter() + .map(|column| { + extract_scalar(row, column) + .with_context(|| format!("row {row_index}: missing choice column `{column}`")) + }) + .collect(); + } + + let column = config + .choices_column + .as_deref() + .context("validated multiple-choice config has no choices source")?; + let value = row + .get(column) + .with_context(|| format!("row {row_index}: missing choices column `{column}`"))?; + let owned; + let value = if let Some(encoded) = value.as_str() { + owned = serde_json::from_str(encoded).unwrap_or_else(|_| value.clone()); + &owned + } else { + value + }; + + match value { + serde_json::Value::Array(values) => values + .iter() + .map(value_to_scalar) + .collect::>>() + .with_context(|| format!("row {row_index}: choices column `{column}` contains non-scalar values")), + serde_json::Value::Object(values) => labels + .iter() + .map(|label| { + values.get(label).and_then(value_to_scalar).with_context(|| { + format!("row {row_index}: choices object `{column}` has no scalar `{label}` value") + }) + }) + .collect(), + _ => bail!("row {row_index}: choices column `{column}` must be an array or object"), + } +} + +fn resolve_correct_index( + row_index: usize, + row: &serde_json::Value, + config: &crate::config::CustomNoCodeQaConfig, + labels: &[String], + choices: &[String], +) -> Result { + if let Some(column) = &config.golden_index_column { + let raw = row + .get(column) + .with_context(|| format!("row {row_index}: missing golden index column `{column}`"))?; + let index = raw + .as_u64() + .or_else(|| raw.as_str().and_then(|value| value.parse().ok())) + .with_context(|| { + format!("row {row_index}: golden index column `{column}` is not an integer") + })?; + let index = usize::try_from(index) + .ok() + .and_then(|index| index.checked_sub(config.golden_index_base)) + .with_context(|| format!("row {row_index}: golden index is below configured base"))?; + if index >= choices.len() { + bail!("row {row_index}: golden index {index} is outside the choice list"); + } + return Ok(index); + } + + if let Some(column) = &config.correct_choice_column { + let columns = config + .choice_columns + .as_ref() + .context("validated correct-choice config has no choice columns")?; + return columns + .iter() + .position(|candidate| candidate == column) + .context("validated correct choice is absent from choice columns"); + } + + let column = config + .golden_column + .as_deref() + .context("validated multiple-choice config has no answer source")?; + let golden = extract_scalar(row, column) + .with_context(|| format!("row {row_index}: missing golden column `{column}`"))?; + labels + .iter() + .position(|label| label.eq_ignore_ascii_case(golden.trim())) + .with_context(|| { + format!("row {row_index}: golden label `{golden}` is not in `choice_labels`") + }) +} + +fn extract_scalar(row: &serde_json::Value, key: &str) -> Option { + row.get(key).and_then(value_to_scalar) +} + +fn value_to_scalar(value: &serde_json::Value) -> Option { + match value { + serde_json::Value::String(value) => Some(value.clone()), + serde_json::Value::Number(value) => Some(value.to_string()), + serde_json::Value::Bool(value) => Some(value.to_string()), + _ => None, + } +} + +fn deterministic_shuffle(values: &mut [T], seed: &str) { + for upper in (1..values.len()).rev() { + let hash = hash_input(&format!("custom-nocode-choice-shuffle-v1:{seed}:{upper}")); + let random = u64::from_str_radix(&hash, 16).unwrap_or_default(); + let index = + usize::try_from(random % u64::try_from(upper + 1).unwrap_or(1)).unwrap_or_default(); + values.swap(upper, index); + } +} + +fn extract_choice_label(response: &str, labels: &[String]) -> Option { + let trimmed = response.trim(); + if let Some(label) = labels + .iter() + .find(|label| label.eq_ignore_ascii_case(trimmed)) + { + return Some(label.clone()); + } + + let tokens: Vec<&str> = trimmed.split_whitespace().collect(); + for token in tokens.iter().rev().take(8) { + let cleaned = token.trim_matches(|character: char| !character.is_ascii_alphanumeric()); + if let Some(label) = labels + .iter() + .find(|label| label.eq_ignore_ascii_case(cleaned)) + { + return Some(label.clone()); + } + } + None } #[cfg(test)] @@ -274,35 +510,133 @@ mod tests { } #[test] - fn render_template_with_prompt_variable() { - let template = "Answer this: {{ prompt }}"; + fn render_template_with_row_variable() { + let template = "Answer this: {{ row.question }}"; let env = jinja::Environment::new(); let rendered = env - .render_str(template, jinja::context!(prompt => "what is 2+2")) + .render_str( + template, + jinja::context!(row => json!({"question": "what is 2+2"})), + ) .unwrap(); assert_eq!(rendered, "Answer this: what is 2+2"); } #[test] fn render_template_preserves_newlines() { - let template = "Question:\n{{ prompt }}\nAnswer:"; + let template = "Question:\n{{ row.question }}\nAnswer:"; let env = jinja::Environment::new(); let rendered = env - .render_str(template, jinja::context!(prompt => "hello")) + .render_str( + template, + jinja::context!(row => json!({"question": "hello"})), + ) .unwrap(); assert_eq!(rendered, "Question:\nhello\nAnswer:"); } #[test] - fn exact_match_case_sensitive() { - assert!(is_exact_match("hello", "hello")); - assert!(!is_exact_match("Hello", "hello")); + fn extracts_multiple_choice_labels() { + let labels = vec!["A".to_owned(), "B".to_owned(), "C".to_owned()]; + assert_eq!(extract_choice_label("B", &labels).as_deref(), Some("B")); + assert_eq!( + extract_choice_label("The answer is (c).", &labels).as_deref(), + Some("C") + ); + assert_eq!(extract_choice_label("unknown", &labels), None); + } + + fn multiple_choice_config() -> crate::config::CustomNoCodeQaConfig { + crate::config::CustomNoCodeQaConfig { + prompt_template_file: "unused.txt".to_owned(), + choice_labels: Some(["A", "B", "C", "D"].map(str::to_owned).to_vec()), + ..crate::config::CustomNoCodeQaConfig::default() + } + } + + #[test] + fn prepares_medqa_object_choices() { + let config = crate::config::CustomNoCodeQaConfig { + choices_column: Some("options".to_owned()), + golden_column: Some("answer_idx".to_owned()), + ..multiple_choice_config() + }; + let row = json!({ + "options": {"A": "alpha", "B": "beta", "C": "gamma", "D": "delta"}, + "answer_idx": "C" + }); + + let prepared = prepare_row(0, &row, &config).unwrap(); + assert_eq!(prepared.golden, "C"); + assert_eq!(prepared.choices[2].text, "gamma"); + } + + #[test] + fn prepares_mmlu_pro_array_choices() { + let config = crate::config::CustomNoCodeQaConfig { + choices_column: Some("options".to_owned()), + golden_column: Some("answer".to_owned()), + ..multiple_choice_config() + }; + let row = json!({"options": ["alpha", "beta", "gamma"], "answer": "B"}); + + let prepared = prepare_row(0, &row, &config).unwrap(); + assert_eq!(prepared.golden, "B"); + assert_eq!(prepared.choices[1].text, "beta"); + } + + #[test] + fn prepares_medmcqa_indexed_choice_columns() { + let config = crate::config::CustomNoCodeQaConfig { + choice_columns: Some(["opa", "opb", "opc", "opd"].map(str::to_owned).to_vec()), + golden_index_column: Some("cop".to_owned()), + ..multiple_choice_config() + }; + let row = json!({"opa": "alpha", "opb": "beta", "opc": "gamma", "opd": "delta", "cop": 1}); + + let prepared = prepare_row(0, &row, &config).unwrap(); + assert_eq!(prepared.golden, "B"); + assert_eq!(prepared.choices[1].text, "beta"); } #[test] - fn exact_match_trims_whitespace() { - assert!(is_exact_match(" hello ", "hello")); - assert!(is_exact_match("hello", " hello ")); + fn prepares_gpqa_with_deterministic_shuffle() { + let config = crate::config::CustomNoCodeQaConfig { + choice_columns: Some( + [ + "Correct Answer", + "Incorrect Answer 1", + "Incorrect Answer 2", + "Incorrect Answer 3", + ] + .map(str::to_owned) + .to_vec(), + ), + correct_choice_column: Some("Correct Answer".to_owned()), + shuffle_choices: true, + shuffle_seed_column: Some("Record ID".to_owned()), + ..multiple_choice_config() + }; + let row = json!({ + "Correct Answer": "correct", + "Incorrect Answer 1": "wrong one", + "Incorrect Answer 2": "wrong two", + "Incorrect Answer 3": "wrong three", + "Record ID": "gpqa-row-1" + }); + + let first = prepare_row(0, &row, &config).unwrap(); + let second = prepare_row(0, &row, &config).unwrap(); + assert_eq!( + serde_json::to_value(&first.choices).unwrap(), + serde_json::to_value(&second.choices).unwrap() + ); + let correct = first + .choices + .iter() + .find(|choice| choice.label == first.golden) + .unwrap(); + assert_eq!(correct.text, "correct"); } #[tokio::test] @@ -321,7 +655,6 @@ mod tests { "style": "qa", "dataset": "fixture/qa", "prompt_template_file": template_path.to_str().unwrap(), - "prompt_column": "q", "golden_column": "a", })) .unwrap(); @@ -393,7 +726,7 @@ mod tests { // Write a Jinja template file. let template_path = root.join("template.txt"); - std::fs::write(&template_path, "{{ prompt }}\nAnswer:").unwrap(); + std::fs::write(&template_path, "{{ row.question }}\nAnswer:").unwrap(); // Pre-populate the dataset cache so no network fetch is needed for rows. let cache = crate::dataset::cache::DatasetCache::new(cache_dir); @@ -411,7 +744,6 @@ mod tests { "dataset": "fixture/qa", "model": "random", "prompt_template_file": template_path.to_str().unwrap(), - "prompt_column": "question", "golden_column": "answer", "limit": 2, })) diff --git a/cli/src/builtins/dataset_runner.rs b/cli/src/builtins/dataset_runner.rs index 1e1b042..efa5618 100644 --- a/cli/src/builtins/dataset_runner.rs +++ b/cli/src/builtins/dataset_runner.rs @@ -81,7 +81,7 @@ impl<'a> DatasetRunner<'a> { &self.info.selected_split, offset, batch_limit, - None, + self.info.revision.as_deref(), ) .await?; @@ -143,6 +143,7 @@ mod tests { available_splits: vec![split.to_string()], selected_split: split.to_string(), config: config.to_string(), + revision: None, } } diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs index 027e287..0b67611 100644 --- a/cli/src/commands/resume.rs +++ b/cli/src/commands/resume.rs @@ -103,13 +103,19 @@ pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<( _ => builtins::resolve(workflow_name) .with_context(|| format!("builtin `{workflow_name}` not found"))?, }; + let custom_nocode_input = match bench_config { + Some(qt::config::BenchmarkConfig::CustomNoCode(config)) => { + Some(super::run::assemble_custom_nocode_input(config, None)) + } + _ => None, + }; super::run::execute_builtin(super::run::ExecuteBuiltinArgs { db: &db, metrics_store: &metrics_store, run_id, workflow_name, builtin, - input: stored_input, + input: custom_nocode_input.as_deref().or(stored_input), json, process_start, }) @@ -219,20 +225,24 @@ mod tests { #[test] fn plan_resume_custom_nocode_with_config() { let file = tempfile::NamedTempFile::new().unwrap(); - let bench = - qt::config::BenchmarkConfig::CustomNoCode(qt::config::CustomNoCodeBenchmarkConfig { + let bench = qt::config::BenchmarkConfig::CustomNoCode(Box::new( + qt::config::CustomNoCodeBenchmarkConfig { type_: "custom_nocode".to_owned(), style: qt::config::CustomNoCodeStyle::Qa, dataset: "quantiles/simpleqa-verified".to_owned(), + dataset_config: None, + split: None, + revision: None, model: Some(qt::llm::Sampler::Random {}), qa: qt::config::CustomNoCodeQaConfig { prompt_template_file: file.path().to_str().unwrap().to_owned(), - prompt_column: "problem".to_owned(), - golden_column: "answer".to_owned(), + golden_column: Some("answer".to_owned()), limit: None, max_workers: None, + ..qt::config::CustomNoCodeQaConfig::default() }, - }); + }, + )); let plan = plan_resume("nocode_custom", &RunStatus::Failed, Some(&bench)).unwrap(); assert!(matches!(plan, ResumePlan::Builtin)); } @@ -240,7 +250,6 @@ mod tests { /// A failed `custom_nocode` run can be resumed and re-execute successfully /// through the `CustomNoCodeBuiltin`, verifying that the resume path wires /// the correct builtin and `ExecuteBuiltinArgs`. - #[expect(clippy::too_many_lines)] #[tokio::test] async fn resume_failed_custom_nocode_run_re_executes_successfully() { use wiremock::matchers::{method, path}; @@ -284,7 +293,7 @@ mod tests { // Write a Jinja template file. let template_path = root.join("template.txt"); - std::fs::write(&template_path, "{{ prompt }}\nAnswer:").unwrap(); + std::fs::write(&template_path, "{{ row.question }}\nAnswer:").unwrap(); // Pre-populate the dataset cache so no network fetch is needed for rows. let cache = qt::dataset::cache::DatasetCache::new(cache_dir); @@ -307,7 +316,6 @@ style = "qa" dataset = "fixture/qa" model = "random" prompt_template_file = "{}" -prompt_column = "question" golden_column = "answer" limit = 2 "#, @@ -316,14 +324,11 @@ limit = 2 ) .unwrap(); + // A started no-code run stores normalized display input, so resume must + // reconstruct the executable configuration from quantiles.toml. let input_json = serde_json::to_string(&serde_json::json!({ - "style": "qa", - "dataset": "fixture/qa", - "model": "random", - "prompt_template_file": template_path.to_str().unwrap(), - "prompt_column": "question", - "golden_column": "answer", - "limit": 2, + "model": "demo-builtin", + "num_samples": 2, })) .unwrap(); diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs index 7bcd87c..f145435 100644 --- a/cli/src/commands/run.rs +++ b/cli/src/commands/run.rs @@ -157,17 +157,49 @@ struct CustomNoCodeConfigInput<'a> { style: &'a str, dataset: &'a str, #[serde(skip_serializing_if = "Option::is_none")] + dataset_config: &'a Option, + #[serde(skip_serializing_if = "Option::is_none")] + split: &'a Option, + #[serde(skip_serializing_if = "Option::is_none")] + revision: &'a Option, + #[serde(skip_serializing_if = "Option::is_none")] model: &'a Option, prompt_template_file: &'a str, - prompt_column: &'a str, - golden_column: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + golden_column: &'a Option, + #[serde(skip_serializing_if = "Option::is_none")] + golden_index_column: &'a Option, + #[serde(skip_serializing_if = "is_zero")] + golden_index_base: usize, + #[serde(skip_serializing_if = "Option::is_none")] + correct_choice_column: &'a Option, + #[serde(skip_serializing_if = "Option::is_none")] + choices_column: &'a Option, + #[serde(skip_serializing_if = "Option::is_none")] + choice_columns: &'a Option>, + #[serde(skip_serializing_if = "Option::is_none")] + choice_labels: &'a Option>, + #[serde(skip_serializing_if = "is_false")] + shuffle_choices: bool, + #[serde(skip_serializing_if = "Option::is_none")] + shuffle_seed_column: &'a Option, #[serde(skip_serializing_if = "Option::is_none")] limit: Option, #[serde(skip_serializing_if = "Option::is_none")] max_workers: Option, } -fn assemble_custom_nocode_input( +#[expect(clippy::trivially_copy_pass_by_ref)] +const fn is_zero(value: &usize) -> bool { + *value == 0 +} + +#[expect(clippy::trivially_copy_pass_by_ref)] +const fn is_false(value: &bool) -> bool { + !*value +} + +pub(super) fn assemble_custom_nocode_input( bench: &qt::config::CustomNoCodeBenchmarkConfig, cli_input: Option<&str>, ) -> String { @@ -178,10 +210,20 @@ fn assemble_custom_nocode_input( let input = CustomNoCodeConfigInput { style: "qa", dataset: &bench.dataset, + dataset_config: &bench.dataset_config, + split: &bench.split, + revision: &bench.revision, model: &bench.model, prompt_template_file: &bench.qa.prompt_template_file, - prompt_column: &bench.qa.prompt_column, golden_column: &bench.qa.golden_column, + golden_index_column: &bench.qa.golden_index_column, + golden_index_base: bench.qa.golden_index_base, + correct_choice_column: &bench.qa.correct_choice_column, + choices_column: &bench.qa.choices_column, + choice_columns: &bench.qa.choice_columns, + choice_labels: &bench.qa.choice_labels, + shuffle_choices: bench.qa.shuffle_choices, + shuffle_seed_column: &bench.qa.shuffle_seed_column, limit: bench.qa.limit, max_workers: bench.qa.max_workers, }; @@ -757,22 +799,27 @@ mod tests { type_: "custom_nocode".to_owned(), style: qt::config::CustomNoCodeStyle::Qa, dataset: "quantiles/simpleqa-verified".to_owned(), + dataset_config: Some("default".to_owned()), + split: Some("test".to_owned()), + revision: Some("main".to_owned()), model: Some(qt::llm::Sampler::Random {}), qa: qt::config::CustomNoCodeQaConfig { prompt_template_file: "prompts/qa.txt".to_owned(), - prompt_column: "problem".to_owned(), - golden_column: "answer".to_owned(), + golden_column: Some("answer".to_owned()), limit: Some(10), max_workers: Some(4), + ..qt::config::CustomNoCodeQaConfig::default() }, }; let input = super::assemble_custom_nocode_input(&bench, None); let parsed: serde_json::Value = serde_json::from_str(&input).unwrap(); assert_eq!(parsed["style"], "qa"); assert_eq!(parsed["dataset"], "quantiles/simpleqa-verified"); + assert_eq!(parsed["dataset_config"], "default"); + assert_eq!(parsed["split"], "test"); + assert_eq!(parsed["revision"], "main"); assert_eq!(parsed["model"], "random"); assert_eq!(parsed["prompt_template_file"], "prompts/qa.txt"); - assert_eq!(parsed["prompt_column"], "problem"); assert_eq!(parsed["golden_column"], "answer"); assert_eq!(parsed["limit"], 10); assert_eq!(parsed["max_workers"], 4); @@ -786,13 +833,16 @@ mod tests { type_: "custom_nocode".to_owned(), style: qt::config::CustomNoCodeStyle::Qa, dataset: "quantiles/simpleqa-verified".to_owned(), + dataset_config: None, + split: None, + revision: None, model: None, qa: qt::config::CustomNoCodeQaConfig { prompt_template_file: "prompts/qa.txt".to_owned(), - prompt_column: "problem".to_owned(), - golden_column: "answer".to_owned(), + golden_column: Some("answer".to_owned()), limit: None, max_workers: None, + ..qt::config::CustomNoCodeQaConfig::default() }, }; let input = super::assemble_custom_nocode_input(&bench, None); @@ -810,13 +860,16 @@ mod tests { type_: "custom_nocode".to_owned(), style: qt::config::CustomNoCodeStyle::Qa, dataset: "quantiles/simpleqa-verified".to_owned(), + dataset_config: None, + split: None, + revision: None, model: None, qa: qt::config::CustomNoCodeQaConfig { prompt_template_file: "prompts/qa.txt".to_owned(), - prompt_column: "problem".to_owned(), - golden_column: "answer".to_owned(), + golden_column: Some("answer".to_owned()), limit: None, max_workers: None, + ..qt::config::CustomNoCodeQaConfig::default() }, }; let input = super::assemble_custom_nocode_input(&bench, Some(r#"{"limit":5}"#)); diff --git a/cli/src/config/custom_nocode.rs b/cli/src/config/custom_nocode.rs index 7028208..4e7c934 100644 --- a/cli/src/config/custom_nocode.rs +++ b/cli/src/config/custom_nocode.rs @@ -1,4 +1,6 @@ -use anyhow::Result; +use std::collections::HashSet; + +use anyhow::{Result, bail}; use serde::{Deserialize, Deserializer}; use crate::llm::Sampler; @@ -34,15 +36,31 @@ impl<'de> Deserialize<'de> for CustomNoCodeStyle { } /// QA-specific configuration for a no-code custom benchmark. -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Clone, Default)] #[serde(deny_unknown_fields)] pub struct CustomNoCodeQaConfig { /// Path to a Jinja template file for rendering prompts. pub prompt_template_file: String, - /// Dataset column containing the prompt text. - pub prompt_column: String, - /// Dataset column containing the golden answer. - pub golden_column: String, + /// Dataset column containing the golden answer label or text. + pub golden_column: Option, + /// Dataset column containing a zero- or one-based golden answer index. + pub golden_index_column: Option, + /// Base used by `golden_index_column`. Defaults to zero. + #[serde(default)] + pub golden_index_base: usize, + /// Dataset column whose value identifies the correct choice before shuffling. + pub correct_choice_column: Option, + /// Dataset column containing choices as an array or object. + pub choices_column: Option, + /// Dataset columns containing choices in their original order. + pub choice_columns: Option>, + /// Labels assigned to choices, for example `["A", "B", "C", "D"]`. + pub choice_labels: Option>, + /// Deterministically shuffle choices before rendering the prompt. + #[serde(default)] + pub shuffle_choices: bool, + /// Dataset column used as the stable seed for choice shuffling. + pub shuffle_seed_column: Option, /// Number of dataset rows to evaluate. pub limit: Option, /// Maximum concurrent workers. @@ -58,12 +76,99 @@ pub struct CustomNoCodeBenchmarkConfig { pub style: CustomNoCodeStyle, /// Dataset identifier in `HuggingFace` (e.g. `quantiles/simpleqa-verified`). pub dataset: String, + /// Optional Hugging Face dataset configuration/subset. + pub dataset_config: Option, + /// Optional dataset split. The dataset manager chooses one when omitted. + pub split: Option, + /// Optional dataset revision. + pub revision: Option, /// Model sampler to use. pub model: Option, #[serde(flatten)] pub qa: CustomNoCodeQaConfig, } +impl CustomNoCodeQaConfig { + /// Validate mutually exclusive answer and choice source fields. + pub(crate) fn validate(&self) -> Result<()> { + let answer_sources = [ + self.golden_column.is_some(), + self.golden_index_column.is_some(), + self.correct_choice_column.is_some(), + ] + .into_iter() + .filter(|configured| *configured) + .count(); + if answer_sources != 1 { + bail!( + "custom_nocode QA config requires exactly one of `golden_column`, `golden_index_column`, or `correct_choice_column`" + ); + } + + let choice_sources = [self.choices_column.is_some(), self.choice_columns.is_some()] + .into_iter() + .filter(|configured| *configured) + .count(); + if choice_sources > 1 { + bail!( + "custom_nocode QA config accepts only one of `choices_column` or `choice_columns`" + ); + } + + let is_multiple_choice = choice_sources == 1; + if (self.golden_index_column.is_some() || self.correct_choice_column.is_some()) + && !is_multiple_choice + { + bail!( + "`golden_index_column` and `correct_choice_column` require a multiple-choice source" + ); + } + if is_multiple_choice && self.choice_labels.as_ref().is_none_or(Vec::is_empty) { + bail!("multiple-choice custom_nocode QA config requires non-empty `choice_labels`"); + } + if !is_multiple_choice && self.choice_labels.is_some() { + bail!("`choice_labels` requires `choices_column` or `choice_columns`"); + } + if self.shuffle_choices && !is_multiple_choice { + bail!("`shuffle_choices` requires `choices_column` or `choice_columns`"); + } + if self.shuffle_choices && self.shuffle_seed_column.is_none() { + bail!("`shuffle_choices = true` requires `shuffle_seed_column`"); + } + if !self.shuffle_choices && self.shuffle_seed_column.is_some() { + bail!("`shuffle_seed_column` requires `shuffle_choices = true`"); + } + if self.golden_index_column.is_none() && self.golden_index_base != 0 { + bail!("`golden_index_base` requires `golden_index_column`"); + } + + if let Some(columns) = &self.choice_columns { + if columns.is_empty() { + bail!("`choice_columns` must not be empty"); + } + if let Some(labels) = &self.choice_labels + && labels.len() != columns.len() + { + bail!("`choice_labels` and `choice_columns` must have the same length"); + } + if let Some(correct) = &self.correct_choice_column + && !columns.contains(correct) + { + bail!("`correct_choice_column` must be present in `choice_columns`"); + } + } + + if let Some(labels) = &self.choice_labels { + let unique: HashSet<&str> = labels.iter().map(String::as_str).collect(); + if unique.len() != labels.len() { + bail!("`choice_labels` must contain unique values"); + } + } + + Ok(()) + } +} + #[cfg(test)] mod tests { use super::super::{BenchmarkConfig, WorkspaceConfig}; @@ -78,7 +183,6 @@ mod tests { dataset = "quantiles/simpleqa-verified" model = "random" prompt_template_file = "prompts/qa.txt" - prompt_column = "problem" golden_column = "answer" limit = 10 "#; @@ -90,8 +194,7 @@ mod tests { assert_eq!(c.dataset, "quantiles/simpleqa-verified"); assert_eq!(c.model, Some(Sampler::Random)); assert_eq!(c.qa.prompt_template_file, "prompts/qa.txt"); - assert_eq!(c.qa.prompt_column, "problem"); - assert_eq!(c.qa.golden_column, "answer"); + assert_eq!(c.qa.golden_column.as_deref(), Some("answer")); assert_eq!(c.qa.limit, Some(10)); } } @@ -105,7 +208,6 @@ mod tests { dataset = "quantiles/simpleqa-verified" model = "random" prompt_template_file = "prompts/qa.txt" - prompt_column = "problem" golden_column = "answer" "#; let result: Result = toml::from_str(toml); @@ -120,7 +222,6 @@ mod tests { style = "qa" dataset = "quantiles/simpleqa-verified" model = "random" - prompt_column = "problem" golden_column = "answer" "#; let result: Result = toml::from_str(toml); @@ -129,19 +230,22 @@ mod tests { #[test] fn validate_rejects_missing_template_file() { - let bench = BenchmarkConfig::CustomNoCode(CustomNoCodeBenchmarkConfig { + let bench = BenchmarkConfig::CustomNoCode(Box::new(CustomNoCodeBenchmarkConfig { type_: "custom_nocode".to_owned(), style: CustomNoCodeStyle::Qa, dataset: "quantiles/simpleqa-verified".to_owned(), + dataset_config: None, + split: None, + revision: None, model: Some(Sampler::Random), qa: CustomNoCodeQaConfig { prompt_template_file: "does_not_exist.txt".to_owned(), - prompt_column: "problem".to_owned(), - golden_column: "answer".to_owned(), + golden_column: Some("answer".to_owned()), limit: None, max_workers: None, + ..CustomNoCodeQaConfig::default() }, - }); + })); let err = bench.validate().unwrap_err(); assert!(err.to_string().contains("not found")); } @@ -149,19 +253,40 @@ mod tests { #[test] fn validate_accepts_existing_template_file() { let file = tempfile::NamedTempFile::new().unwrap(); - let bench = BenchmarkConfig::CustomNoCode(CustomNoCodeBenchmarkConfig { + let bench = BenchmarkConfig::CustomNoCode(Box::new(CustomNoCodeBenchmarkConfig { type_: "custom_nocode".to_owned(), style: CustomNoCodeStyle::Qa, dataset: "quantiles/simpleqa-verified".to_owned(), + dataset_config: None, + split: None, + revision: None, model: Some(Sampler::Random), qa: CustomNoCodeQaConfig { prompt_template_file: file.path().to_str().unwrap().to_owned(), - prompt_column: "problem".to_owned(), - golden_column: "answer".to_owned(), + golden_column: Some("answer".to_owned()), limit: None, max_workers: None, + ..CustomNoCodeQaConfig::default() }, - }); + })); bench.validate().unwrap(); } + + #[test] + fn parses_all_custom_nocode_examples() { + let config: WorkspaceConfig = toml::from_str(include_str!( + "../../../custom-nocode-examples/quantiles.toml" + )) + .unwrap(); + + for name in ["nocode_custom", "medqa", "medmcqa", "mmlu-pro", "gpqa"] { + assert!( + matches!( + config.benchmarks.get(name), + Some(BenchmarkConfig::CustomNoCode(_)) + ), + "missing custom_nocode example `{name}`" + ); + } + } } diff --git a/cli/src/config/mod.rs b/cli/src/config/mod.rs index 9dd9dc6..86c32df 100644 --- a/cli/src/config/mod.rs +++ b/cli/src/config/mod.rs @@ -13,7 +13,7 @@ use crate::llm::Sampler; pub enum BenchmarkConfig { Builtin(BuiltinBenchmarkConfig), CustomCode(CustomCodeBenchmarkConfig), - CustomNoCode(CustomNoCodeBenchmarkConfig), + CustomNoCode(Box), } impl BenchmarkConfig { @@ -38,6 +38,7 @@ impl BenchmarkConfig { c.qa.prompt_template_file ); } + c.qa.validate()?; Ok(()) } } @@ -69,7 +70,7 @@ impl<'de> Deserialize<'de> for BenchmarkConfig { "failed to deserialize custom_nocode benchmark config: {e}" )) })?; - Ok(BenchmarkConfig::CustomNoCode(config)) + Ok(BenchmarkConfig::CustomNoCode(Box::new(config))) } Some("builtin") | None => { let config = BuiltinBenchmarkConfig::deserialize(value).map_err(|e| { diff --git a/cli/src/dataset/cache.rs b/cli/src/dataset/cache.rs index 9b7b674..4c6c3e8 100644 --- a/cli/src/dataset/cache.rs +++ b/cli/src/dataset/cache.rs @@ -6,11 +6,14 @@ use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use serde_json::Value; use sha2::{Digest, Sha256}; -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use std::fs::File; use std::path::Path; use std::sync::Arc; +const JSON_ENCODED_METADATA_KEY: &str = "quantiles.json_encoded"; +const CACHE_FORMAT_VERSION: &str = "v2"; + /// Manages the on-disk cache for dataset batches. pub struct DatasetCache { root: std::path::PathBuf, @@ -81,7 +84,7 @@ impl DatasetCache { let mut obj = serde_json::Map::new(); for (col_idx, field) in schema.fields().iter().enumerate() { let col = batch.column(col_idx); - let val = arrow_value_to_json(col, row_idx)?; + let val = arrow_value_to_json(col, field, row_idx)?; obj.insert(field.name().clone(), val); } rows.push(Value::Object(obj)); @@ -95,6 +98,8 @@ impl DatasetCache { #[must_use] pub fn cache_key(dataset_id: &str, config: &str, split: &str, revision: Option<&str>) -> String { let mut hasher = Sha256::new(); + hasher.update(CACHE_FORMAT_VERSION.as_bytes()); + hasher.update(b":"); hasher.update(dataset_id.as_bytes()); hasher.update(b":"); hasher.update(config.as_bytes()); @@ -128,8 +133,18 @@ fn json_to_arrow(rows: &[Value]) -> Result { .map(|r| r.get(&key).unwrap_or(&Value::Null)) .collect(); let dtype = infer_type(&values); - fields.push(Field::new(key.clone(), dtype.clone(), true)); - arrays.push(build_array(&values, &dtype)?); + let mut field = Field::new(key.clone(), dtype.clone(), true); + let json_encoded = values + .iter() + .any(|value| matches!(value, Value::Array(_) | Value::Object(_))); + if json_encoded { + field = field.with_metadata(HashMap::from([( + JSON_ENCODED_METADATA_KEY.to_owned(), + "true".to_owned(), + )])); + } + fields.push(field); + arrays.push(build_array(&values, &dtype, json_encoded)?); } let schema = Arc::new(Schema::new(fields)); @@ -172,7 +187,7 @@ fn infer_type(values: &[&Value]) -> DataType { } } -fn build_array(values: &[&Value], dtype: &DataType) -> Result { +fn build_array(values: &[&Value], dtype: &DataType, json_encoded: bool) -> Result { match dtype { DataType::Boolean => { let mut builder = BooleanBuilder::new(); @@ -208,8 +223,9 @@ fn build_array(values: &[&Value], dtype: &DataType) -> Result { let mut builder = StringBuilder::new(); for v in values { match v { - Value::String(s) => builder.append_value(s), Value::Null => builder.append_null(), + value if json_encoded => builder.append_value(serde_json::to_string(value)?), + Value::String(s) => builder.append_value(s), other => builder.append_value(other.to_string()), } } @@ -219,7 +235,7 @@ fn build_array(values: &[&Value], dtype: &DataType) -> Result { } } -fn arrow_value_to_json(col: &dyn arrow::array::Array, row: usize) -> Result { +fn arrow_value_to_json(col: &dyn arrow::array::Array, field: &Field, row: usize) -> Result { use arrow::array::{BooleanArray, Float64Array, Int64Array, StringArray}; if col.is_null(row) { @@ -227,7 +243,16 @@ fn arrow_value_to_json(col: &dyn arrow::array::Array, row: usize) -> Result() { - Ok(Value::String(arr.value(row).to_string())) + let value = arr.value(row); + if field + .metadata() + .get(JSON_ENCODED_METADATA_KEY) + .is_some_and(|encoded| encoded == "true") + { + serde_json::from_str(value).context("failed to decode cached JSON column") + } else { + Ok(Value::String(value.to_string())) + } } else if let Some(arr) = col.as_any().downcast_ref::() { let n: i64 = arr.value(row); Ok(Value::Number(n.into())) @@ -291,6 +316,25 @@ mod tests { assert_eq!(read[2]["name"], "charlie"); } + #[tokio::test] + async fn test_nested_json_roundtrip() { + let tmpdir = tempfile::tempdir().unwrap(); + let cache = DatasetCache::new(tmpdir.path().to_path_buf()); + let path = cache.batch_path("nested", 0, 10); + let rows = vec![ + json!({ + "options": {"A": "alpha", "B": "beta"}, + "list": ["one", "two"] + }), + json!({"options": "unstructured", "list": ["three"]}), + ]; + + cache.write_batch(&path, &rows).await.unwrap(); + let read = cache.read_batch(&path).await.unwrap(); + + assert_eq!(read, rows); + } + #[tokio::test] async fn test_empty_batch_roundtrip() { let tmpdir = tempfile::tempdir().unwrap(); diff --git a/cli/src/dataset/mod.rs b/cli/src/dataset/mod.rs index fa78209..13be19d 100644 --- a/cli/src/dataset/mod.rs +++ b/cli/src/dataset/mod.rs @@ -14,6 +14,7 @@ pub struct DatasetInfo { pub available_splits: Vec, pub selected_split: String, pub config: String, + pub revision: Option, } /// Central manager that coordinates fetching from huggingface and local caching. @@ -105,6 +106,7 @@ impl DatasetManager { available_splits: splits, selected_split, config, + revision: revision.map(str::to_owned), }) } diff --git a/custom-nocode-examples/prompts/gpqa.txt b/custom-nocode-examples/prompts/gpqa.txt new file mode 100644 index 0000000..dfd58b0 --- /dev/null +++ b/custom-nocode-examples/prompts/gpqa.txt @@ -0,0 +1,5 @@ +{{ row["Question"] }} + +{% for choice in choices %}{{ choice.label }}. {{ choice.text }} +{% endfor %} +Answer with only the letter of the correct choice. diff --git a/custom-nocode-examples/prompts/medmcqa.txt b/custom-nocode-examples/prompts/medmcqa.txt new file mode 100644 index 0000000..7889828 --- /dev/null +++ b/custom-nocode-examples/prompts/medmcqa.txt @@ -0,0 +1,5 @@ +{{ row.question }} + +{% for choice in choices %}{{ choice.label }}. {{ choice.text }} +{% endfor %} +Answer with only the letter of the correct choice. diff --git a/custom-nocode-examples/prompts/medqa.txt b/custom-nocode-examples/prompts/medqa.txt new file mode 100644 index 0000000..7889828 --- /dev/null +++ b/custom-nocode-examples/prompts/medqa.txt @@ -0,0 +1,5 @@ +{{ row.question }} + +{% for choice in choices %}{{ choice.label }}. {{ choice.text }} +{% endfor %} +Answer with only the letter of the correct choice. diff --git a/custom-nocode-examples/prompts/mmlu-pro.txt b/custom-nocode-examples/prompts/mmlu-pro.txt new file mode 100644 index 0000000..7889828 --- /dev/null +++ b/custom-nocode-examples/prompts/mmlu-pro.txt @@ -0,0 +1,5 @@ +{{ row.question }} + +{% for choice in choices %}{{ choice.label }}. {{ choice.text }} +{% endfor %} +Answer with only the letter of the correct choice. diff --git a/custom-nocode-examples/prompts/qa.txt b/custom-nocode-examples/prompts/qa.txt index c4f9438..87f2395 100644 --- a/custom-nocode-examples/prompts/qa.txt +++ b/custom-nocode-examples/prompts/qa.txt @@ -1 +1 @@ -{{ prompt }} +{{ row.problem }} diff --git a/custom-nocode-examples/quantiles.toml b/custom-nocode-examples/quantiles.toml index 1372044..5879207 100644 --- a/custom-nocode-examples/quantiles.toml +++ b/custom-nocode-examples/quantiles.toml @@ -4,6 +4,60 @@ style = "qa" dataset = "quantiles/simpleqa-verified" model = "random" prompt_template_file = "prompts/qa.txt" -prompt_column = "problem" golden_column = "answer" limit = 10 + +[benchmarks.medqa] +type = "custom_nocode" +style = "qa" +dataset = "GBaker/MedQA-USMLE-4-options" +dataset_config = "default" +split = "test" +model = "random" +prompt_template_file = "prompts/medqa.txt" +choices_column = "options" +choice_labels = ["A", "B", "C", "D"] +golden_column = "answer_idx" +limit = 10 + +[benchmarks.medmcqa] +type = "custom_nocode" +style = "qa" +dataset = "openlifescienceai/medmcqa" +dataset_config = "default" +split = "test" +model = "random" +prompt_template_file = "prompts/medmcqa.txt" +choice_columns = ["opa", "opb", "opc", "opd"] +choice_labels = ["A", "B", "C", "D"] +golden_index_column = "cop" +golden_index_base = 0 +limit = 10 + +[benchmarks.mmlu-pro] +type = "custom_nocode" +style = "qa" +dataset = "TIGER-Lab/MMLU-Pro" +dataset_config = "default" +split = "test" +model = "random" +prompt_template_file = "prompts/mmlu-pro.txt" +choices_column = "options" +choice_labels = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"] +golden_column = "answer" +limit = 10 + +[benchmarks.gpqa] +type = "custom_nocode" +style = "qa" +dataset = "Wanfq/gpqa" +dataset_config = "gpqa_diamond" +split = "train" +model = "random" +prompt_template_file = "prompts/gpqa.txt" +choice_columns = ["Correct Answer", "Incorrect Answer 1", "Incorrect Answer 2", "Incorrect Answer 3"] +choice_labels = ["A", "B", "C", "D"] +correct_choice_column = "Correct Answer" +shuffle_choices = true +shuffle_seed_column = "Record ID" +limit = 10 From 3632f22305541f8a2da14d778f5fa937712000a0 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:34:29 -0700 Subject: [PATCH 02/16] progress --- CONFIG.md | 38 +-- cli/README.md | 6 +- cli/src/builtins/custom_nocode.rs | 279 ++++++++++++-------- cli/src/commands/resume.rs | 24 +- cli/src/commands/run.rs | 129 +++------ cli/src/config/custom_nocode.rs | 366 ++++++++++++++------------ cli/src/config/mod.rs | 6 +- custom-nocode-examples/quantiles.toml | 48 ++-- 8 files changed, 476 insertions(+), 420 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index b851941..d71a524 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -8,7 +8,7 @@ You need a config file when you want to do one or more of the following: - Override built-in benchmark defaults (e.g. model, sample limit) - Define custom evaluations (`type = "custom_code"`) -- Define no-code QA benchmarks (`type = "custom_nocode"`, `style = "qa"`) +- Define no-code QA benchmarks (`type = "custom_nocode"`) - Resume custom evaluations later with `qt resume ` You don't, however, need any configuration when you want to run built-in benchmarks. `qt run pubmedqa`, for example, works out of the box. @@ -100,13 +100,13 @@ Note that models require specific configuration based on the provider. For detai ### `custom_nocode` -No-code benchmarks are configured in TOML and run natively inside the CLI, without a custom Python or TypeScript evaluation program. The initial supported style is `qa`, which renders a prompt from a dataset row, calls a model, and scores the response with exact-match accuracy against the configured golden answer column. +No-code benchmarks are configured in TOML and run natively inside the CLI, without a custom Python or TypeScript evaluation program. The `exact_match` style scores an open answer or label against a golden answer column. The `multiple_choice` style normalizes choices, extracts the selected label from the response, and scores it against a configured label, index, or correct-choice column. ```toml [benchmarks.nocode_custom] type = "custom_nocode" -style = "qa" -dataset = "quantiles/simpleqa-verified" +style = "exact_match" +dataset = { name = "quantiles/simpleqa-verified" } model = "random" prompt_template_file = "prompts/qa.txt" golden_column = "answer" @@ -124,22 +124,26 @@ For a complete minimal example, see [`custom-nocode-examples/quantiles.toml`](./ | Field | Type | Required | Description | |--|--|--|--| | `type` | string | yes | Must be `"custom_nocode"`. | -| `style` | string | yes | Must be `"qa"`. | -| `dataset` | string | yes | Dataset identifier, for example `"quantiles/simpleqa-verified"`. | -| `dataset_config` | string | no | Hugging Face dataset configuration or subset. | -| `split` | string | no | Dataset split. When omitted, Quantiles selects a standard evaluation split. | -| `revision` | string | no | Dataset revision. | +| `style` | string | yes | `"exact_match"` for open-answer or label exact match, or `"multiple_choice"` for choice-based benchmarks. | +| `dataset` | table | yes | Hugging Face dataset coordinates. | +| `dataset.name` | string | yes | Dataset identifier, for example `"quantiles/simpleqa-verified"`. | +| `dataset.config_name` | string | no | Hugging Face dataset configuration or subset. | +| `dataset.split` | string | no | Dataset split. When omitted, Quantiles selects a standard evaluation split. | +| `dataset.revision` | string | no | Dataset revision. | | `model` | string or table | no | Model sampler. Defaults to the demo random sampler. See [model naming](#model-naming). | | `prompt_template_file` | string | yes | Path to a Jinja prompt template file. The template receives the complete dataset `row` and, for multiple-choice benchmarks, normalized `choices`. | -| `golden_column` | string | conditional | Dataset column containing the golden answer text or label. Exactly one golden answer source is required. | -| `golden_index_column` | string | conditional | Dataset column containing the golden choice index. | -| `golden_index_base` | integer | no | Index base for `golden_index_column`. Defaults to `0`. | -| `correct_choice_column` | string | conditional | Choice column known to contain the correct answer, useful when choices are shuffled. | -| `choices_column` | string | no | Dataset column containing choices as an array or label-keyed object. | -| `choice_columns` | array of strings | no | Dataset columns containing choices in their original order. | +| `golden_column` | string | conditional | Dataset column containing the golden answer. Required for `exact_match`. | +| `choices` | table | conditional | Choice source. Required for `multiple_choice`. Configure exactly one of `choices.column` or `choices.columns`. | +| `choices.column` | string | conditional | Dataset column containing choices as an array or label-keyed object. | +| `choices.columns` | array of strings | conditional | Dataset columns containing choices in their original order. | +| `answer` | table | conditional | Correct-answer source. Required for `multiple_choice`. Configure exactly one answer-source form. | +| `answer.label_column` | string | conditional | Dataset column containing the golden choice label. | +| `answer.index_column` | string | conditional | Dataset column containing the golden choice index. | +| `answer.index_base` | integer | no | Index base for `answer.index_column`. Defaults to `0`. | +| `answer.correct_choice_column` | string | conditional | Member of `choices.columns` known to contain the correct answer. | | `choice_labels` | array of strings | conditional | Labels assigned to choices in order. Required for multiple choice; array-backed rows may use a prefix of the configured labels. | -| `shuffle_choices` | boolean | no | Deterministically shuffle choices. Defaults to `false`. | -| `shuffle_seed_column` | string | conditional | Stable row identifier used when `shuffle_choices = true`. | +| `shuffle` | table | no | Enables deterministic choice shuffling for `multiple_choice`. | +| `shuffle.seed_column` | string | conditional | Stable row identifier used to seed deterministic shuffling. Required when `shuffle` is present. | | `limit` | integer | no | Number of dataset rows to evaluate. | | `max_workers` | integer | no | Maximum concurrent workers. | diff --git a/cli/README.md b/cli/README.md index 68578e1..9dadecf 100644 --- a/cli/README.md +++ b/cli/README.md @@ -73,13 +73,13 @@ max_workers = 100 ### No-code QA benchmarks -For dataset-backed QA checks that do not need custom evaluation code, set `type = "custom_nocode"` and `style = "qa"`. The benchmark runs inside the CLI, renders each prompt with the configured Jinja template, calls the configured model, and scores each row with exact-match accuracy against the golden answer column. +For dataset-backed QA checks that do not need custom evaluation code, set `type = "custom_nocode"`. Use `style = "exact_match"` for benchmarks that test an exact match to a golden answer, and `style = "multiple_choice"` for choice-based benchmarks. The benchmark runs inside the CLI, renders each prompt with the configured Jinja template, calls the configured model, and scores each row against the configured answer source. ```toml [benchmarks.nocode_custom] type = "custom_nocode" -style = "qa" -dataset = "quantiles/simpleqa-verified" +style = "exact_match" +dataset = { name = "quantiles/simpleqa-verified" } model = "random" prompt_template_file = "prompts/qa.txt" golden_column = "answer" diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index a69cc03..44bda97 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -16,15 +16,13 @@ use crate::llm::random::RandomSampler; /// Input deserialized from the JSON assembled by `commands::run`. #[derive(Debug, Deserialize)] struct CustomNoCodeInput { - style: crate::config::CustomNoCodeStyle, - dataset: String, - dataset_config: Option, - split: Option, - revision: Option, + dataset: crate::config::CustomNoCodeDatasetConfig, #[serde(default)] model: Option, + limit: Option, + max_workers: Option, #[serde(flatten)] - qa: crate::config::CustomNoCodeQaConfig, + task: crate::config::CustomNoCodeTaskConfig, } /// Per-row step output stored as JSON in the step record. @@ -55,8 +53,8 @@ struct EvaluateRowArgs<'a> { i: usize, /// The row value row: &'a serde_json::Value, - /// QA and multiple-choice configuration. - qa: &'a crate::config::CustomNoCodeQaConfig, + /// Exact-match or multiple-choice task configuration. + task: &'a crate::config::CustomNoCodeTaskConfig, /// The prompt template template_str: &'a str, /// The pre-constructed jinja template env @@ -86,7 +84,7 @@ impl CustomNoCodeBuiltin { } async fn evaluate_row(&self, args: EvaluateRowArgs<'_>) -> Result { - let prepared = prepare_row(args.i, args.row, args.qa)?; + let prepared = prepare_row(args.i, args.row, args.task)?; let rendered = args .env @@ -118,10 +116,13 @@ impl CustomNoCodeBuiltin { .with_context(|| format!("failed to sample LLM for row {}", args.i))?; let parsed_response = if prepared.is_multiple_choice { - extract_choice_label( - &model_response, - args.qa.choice_labels.as_deref().unwrap_or_default(), - ) + let crate::config::CustomNoCodeTaskConfig::MultipleChoice { + choice_labels, .. + } = args.task + else { + unreachable!("prepared multiple-choice row has exact-match config") + }; + extract_choice_label(&model_response, choice_labels) } else { Some(model_response.trim().to_owned()) }; @@ -162,16 +163,16 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { async fn execute(&self, ctx: BuiltinContext<'_>) -> Result<()> { let config = parse_input(ctx.input)?; - let (template_str, env) = load_template(&config.qa.prompt_template_file)?; - let max_workers = config.qa.max_workers.unwrap_or_else(get_max_workers); + let (template_str, env) = load_template(config.task.prompt_template_file())?; + let max_workers = config.max_workers.unwrap_or_else(get_max_workers); let llm = resolve_sampler(config.model.as_ref(), || Arc::new(RandomSampler::new(80)))?; let (manager, info, limit) = resolve_dataset_limit( - &config.dataset, - config.dataset_config.as_deref(), - config.split.as_deref(), - config.revision.as_deref(), - config.qa.limit, + &config.dataset.name, + config.dataset.config_name.as_deref(), + config.dataset.split.as_deref(), + config.dataset.revision.as_deref(), + config.limit, ) .await?; set_builtin_run_input( @@ -179,7 +180,7 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { ctx.run_id, config.model.as_ref(), limit, - config.qa.max_workers, + config.max_workers, ) .await?; @@ -189,8 +190,8 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { .as_ref() .map_or("random".to_string(), std::string::ToString::to_string); let run_id = ctx.run_id; - let dataset = config.dataset.clone(); - let qa = Arc::new(config.qa.clone()); + let dataset = config.dataset.name.clone(); + let task = Arc::new(config.task.clone()); let template_str = Arc::new(template_str); let name = self.name(); @@ -202,13 +203,13 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { let db = db.clone(); let model_name = model_name.clone(); let template_str = Arc::clone(&template_str); - let qa = Arc::clone(&qa); + let task = Arc::clone(&task); let env = env.clone(); async move { let args = EvaluateRowArgs { i, row: &row, - qa: &qa, + task: &task, template_str: &template_str, env: &env, model_name: &model_name, @@ -238,11 +239,7 @@ fn parse_input(input: Option<&str>) -> Result { .context("invalid builtin input JSON")? .context("custom_nocode benchmark requires input configuration")?; - match config.style { - crate::config::CustomNoCodeStyle::Qa => {} - } - - if config.qa.limit == Some(0) { + if config.limit == Some(0) { bail!("limit must be > 0"); } @@ -284,15 +281,21 @@ async fn resolve_dataset_limit( fn prepare_row( row_index: usize, row: &serde_json::Value, - config: &crate::config::CustomNoCodeQaConfig, + task: &crate::config::CustomNoCodeTaskConfig, ) -> Result { - let Some(labels) = config.choice_labels.as_ref() else { - let column = config - .golden_column - .as_deref() - .context("validated custom_nocode config has no golden column")?; - let golden = extract_scalar(row, column) - .with_context(|| format!("row {row_index}: missing golden column `{column}`"))?; + let crate::config::CustomNoCodeTaskConfig::MultipleChoice { + choices: choice_source, + answer, + choice_labels, + shuffle, + .. + } = task + else { + let crate::config::CustomNoCodeTaskConfig::ExactMatch { golden_column, .. } = task else { + unreachable!() + }; + let golden = extract_scalar(row, golden_column) + .with_context(|| format!("row {row_index}: missing golden column `{golden_column}`"))?; return Ok(PreparedRow { choices: Vec::new(), golden, @@ -300,30 +303,36 @@ fn prepare_row( }); }; - let choice_values = extract_choices(row_index, row, config, labels)?; - if choice_values.is_empty() || choice_values.len() > labels.len() { + let choice_values = extract_choices(row_index, row, choice_source, choice_labels)?; + if choice_values.is_empty() || choice_values.len() > choice_labels.len() { bail!( "row {row_index}: found {} choices but configured only {} labels", choice_values.len(), - labels.len() + choice_labels.len() ); } - let labels = &labels[..choice_values.len()]; - - let correct_index = resolve_correct_index(row_index, row, config, labels, &choice_values)?; + let labels = &choice_labels[..choice_values.len()]; + + let correct_index = resolve_correct_index( + row_index, + row, + choice_source, + answer, + labels, + &choice_values, + )?; let mut indexed_choices: Vec<(String, bool)> = choice_values .into_iter() .enumerate() .map(|(index, text)| (text, index == correct_index)) .collect(); - if config.shuffle_choices { - let seed_column = config - .shuffle_seed_column - .as_deref() - .context("validated shuffle config has no seed column")?; - let seed = extract_scalar(row, seed_column).with_context(|| { - format!("row {row_index}: missing shuffle seed column `{seed_column}`") + if let Some(shuffle) = shuffle { + let seed = extract_scalar(row, &shuffle.seed_column).with_context(|| { + format!( + "row {row_index}: missing shuffle seed column `{}`", + shuffle.seed_column + ) })?; deterministic_shuffle(&mut indexed_choices, &seed); } @@ -353,10 +362,13 @@ fn prepare_row( fn extract_choices( row_index: usize, row: &serde_json::Value, - config: &crate::config::CustomNoCodeQaConfig, + source: &crate::config::CustomNoCodeChoiceSource, labels: &[String], ) -> Result> { - if let Some(columns) = &config.choice_columns { + if let crate::config::CustomNoCodeChoiceSource::Columns( + crate::config::CustomNoCodeChoiceColumns { columns }, + ) = source + { return columns .iter() .map(|column| { @@ -366,10 +378,12 @@ fn extract_choices( .collect(); } - let column = config - .choices_column - .as_deref() - .context("validated multiple-choice config has no choices source")?; + let crate::config::CustomNoCodeChoiceSource::Column(crate::config::CustomNoCodeChoiceColumn { + column, + }) = source + else { + unreachable!() + }; let value = row .get(column) .with_context(|| format!("row {row_index}: missing choices column `{column}`"))?; @@ -402,11 +416,18 @@ fn extract_choices( fn resolve_correct_index( row_index: usize, row: &serde_json::Value, - config: &crate::config::CustomNoCodeQaConfig, + choice_source: &crate::config::CustomNoCodeChoiceSource, + answer: &crate::config::CustomNoCodeAnswerSource, labels: &[String], choices: &[String], ) -> Result { - if let Some(column) = &config.golden_index_column { + if let crate::config::CustomNoCodeAnswerSource::IndexColumn( + crate::config::CustomNoCodeIndexAnswer { + index_column: column, + index_base, + }, + ) = answer + { let raw = row .get(column) .with_context(|| format!("row {row_index}: missing golden index column `{column}`"))?; @@ -418,7 +439,7 @@ fn resolve_correct_index( })?; let index = usize::try_from(index) .ok() - .and_then(|index| index.checked_sub(config.golden_index_base)) + .and_then(|index| index.checked_sub(*index_base)) .with_context(|| format!("row {row_index}: golden index is below configured base"))?; if index >= choices.len() { bail!("row {row_index}: golden index {index} is outside the choice list"); @@ -426,21 +447,32 @@ fn resolve_correct_index( return Ok(index); } - if let Some(column) = &config.correct_choice_column { - let columns = config - .choice_columns - .as_ref() - .context("validated correct-choice config has no choice columns")?; + if let crate::config::CustomNoCodeAnswerSource::CorrectChoiceColumn( + crate::config::CustomNoCodeCorrectChoiceAnswer { + correct_choice_column: column, + }, + ) = answer + { + let crate::config::CustomNoCodeChoiceSource::Columns( + crate::config::CustomNoCodeChoiceColumns { columns }, + ) = choice_source + else { + bail!("correct-choice answer requires column-backed choices"); + }; return columns .iter() .position(|candidate| candidate == column) .context("validated correct choice is absent from choice columns"); } - let column = config - .golden_column - .as_deref() - .context("validated multiple-choice config has no answer source")?; + let crate::config::CustomNoCodeAnswerSource::LabelColumn( + crate::config::CustomNoCodeLabelAnswer { + label_column: column, + }, + ) = answer + else { + unreachable!() + }; let golden = extract_scalar(row, column) .with_context(|| format!("row {row_index}: missing golden column `{column}`"))?; labels @@ -546,21 +578,33 @@ mod tests { assert_eq!(extract_choice_label("unknown", &labels), None); } - fn multiple_choice_config() -> crate::config::CustomNoCodeQaConfig { - crate::config::CustomNoCodeQaConfig { + fn multiple_choice_config( + choices: crate::config::CustomNoCodeChoiceSource, + answer: crate::config::CustomNoCodeAnswerSource, + ) -> crate::config::CustomNoCodeTaskConfig { + crate::config::CustomNoCodeTaskConfig::MultipleChoice { prompt_template_file: "unused.txt".to_owned(), - choice_labels: Some(["A", "B", "C", "D"].map(str::to_owned).to_vec()), - ..crate::config::CustomNoCodeQaConfig::default() + choices, + answer, + choice_labels: ["A", "B", "C", "D"].map(str::to_owned).to_vec(), + shuffle: None, } } #[test] fn prepares_medqa_object_choices() { - let config = crate::config::CustomNoCodeQaConfig { - choices_column: Some("options".to_owned()), - golden_column: Some("answer_idx".to_owned()), - ..multiple_choice_config() - }; + let config = multiple_choice_config( + crate::config::CustomNoCodeChoiceSource::Column( + crate::config::CustomNoCodeChoiceColumn { + column: "options".to_owned(), + }, + ), + crate::config::CustomNoCodeAnswerSource::LabelColumn( + crate::config::CustomNoCodeLabelAnswer { + label_column: "answer_idx".to_owned(), + }, + ), + ); let row = json!({ "options": {"A": "alpha", "B": "beta", "C": "gamma", "D": "delta"}, "answer_idx": "C" @@ -573,11 +617,18 @@ mod tests { #[test] fn prepares_mmlu_pro_array_choices() { - let config = crate::config::CustomNoCodeQaConfig { - choices_column: Some("options".to_owned()), - golden_column: Some("answer".to_owned()), - ..multiple_choice_config() - }; + let config = multiple_choice_config( + crate::config::CustomNoCodeChoiceSource::Column( + crate::config::CustomNoCodeChoiceColumn { + column: "options".to_owned(), + }, + ), + crate::config::CustomNoCodeAnswerSource::LabelColumn( + crate::config::CustomNoCodeLabelAnswer { + label_column: "answer".to_owned(), + }, + ), + ); let row = json!({"options": ["alpha", "beta", "gamma"], "answer": "B"}); let prepared = prepare_row(0, &row, &config).unwrap(); @@ -587,11 +638,19 @@ mod tests { #[test] fn prepares_medmcqa_indexed_choice_columns() { - let config = crate::config::CustomNoCodeQaConfig { - choice_columns: Some(["opa", "opb", "opc", "opd"].map(str::to_owned).to_vec()), - golden_index_column: Some("cop".to_owned()), - ..multiple_choice_config() - }; + let config = multiple_choice_config( + crate::config::CustomNoCodeChoiceSource::Columns( + crate::config::CustomNoCodeChoiceColumns { + columns: ["opa", "opb", "opc", "opd"].map(str::to_owned).to_vec(), + }, + ), + crate::config::CustomNoCodeAnswerSource::IndexColumn( + crate::config::CustomNoCodeIndexAnswer { + index_column: "cop".to_owned(), + index_base: 0, + }, + ), + ); let row = json!({"opa": "alpha", "opb": "beta", "opc": "gamma", "opd": "delta", "cop": 1}); let prepared = prepare_row(0, &row, &config).unwrap(); @@ -601,21 +660,29 @@ mod tests { #[test] fn prepares_gpqa_with_deterministic_shuffle() { - let config = crate::config::CustomNoCodeQaConfig { - choice_columns: Some( - [ - "Correct Answer", - "Incorrect Answer 1", - "Incorrect Answer 2", - "Incorrect Answer 3", - ] - .map(str::to_owned) - .to_vec(), + let config = crate::config::CustomNoCodeTaskConfig::MultipleChoice { + prompt_template_file: "unused.txt".to_owned(), + choices: crate::config::CustomNoCodeChoiceSource::Columns( + crate::config::CustomNoCodeChoiceColumns { + columns: [ + "Correct Answer", + "Incorrect Answer 1", + "Incorrect Answer 2", + "Incorrect Answer 3", + ] + .map(str::to_owned) + .to_vec(), + }, + ), + answer: crate::config::CustomNoCodeAnswerSource::CorrectChoiceColumn( + crate::config::CustomNoCodeCorrectChoiceAnswer { + correct_choice_column: "Correct Answer".to_owned(), + }, ), - correct_choice_column: Some("Correct Answer".to_owned()), - shuffle_choices: true, - shuffle_seed_column: Some("Record ID".to_owned()), - ..multiple_choice_config() + choice_labels: ["A", "B", "C", "D"].map(str::to_owned).to_vec(), + shuffle: Some(crate::config::CustomNoCodeShuffleConfig { + seed_column: "Record ID".to_owned(), + }), }; let row = json!({ "Correct Answer": "correct", @@ -652,8 +719,8 @@ mod tests { std::fs::write(&template_path, "{{ unclosed").unwrap(); let input_json = serde_json::to_string(&json!({ - "style": "qa", - "dataset": "fixture/qa", + "style": "exact_match", + "dataset": {"name": "fixture/qa"}, "prompt_template_file": template_path.to_str().unwrap(), "golden_column": "a", })) @@ -740,8 +807,8 @@ mod tests { // Assemble the input JSON that execute() expects. let input_json = serde_json::to_string(&json!({ - "style": "qa", - "dataset": "fixture/qa", + "style": "exact_match", + "dataset": {"name": "fixture/qa"}, "model": "random", "prompt_template_file": template_path.to_str().unwrap(), "golden_column": "answer", diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs index 0b67611..bf0de50 100644 --- a/cli/src/commands/resume.rs +++ b/cli/src/commands/resume.rs @@ -228,18 +228,18 @@ mod tests { let bench = qt::config::BenchmarkConfig::CustomNoCode(Box::new( qt::config::CustomNoCodeBenchmarkConfig { type_: "custom_nocode".to_owned(), - style: qt::config::CustomNoCodeStyle::Qa, - dataset: "quantiles/simpleqa-verified".to_owned(), - dataset_config: None, - split: None, - revision: None, + dataset: qt::config::CustomNoCodeDatasetConfig { + name: "quantiles/simpleqa-verified".to_owned(), + config_name: None, + split: None, + revision: None, + }, model: Some(qt::llm::Sampler::Random {}), - qa: qt::config::CustomNoCodeQaConfig { + limit: None, + max_workers: None, + task: qt::config::CustomNoCodeTaskConfig::ExactMatch { prompt_template_file: file.path().to_str().unwrap().to_owned(), - golden_column: Some("answer".to_owned()), - limit: None, - max_workers: None, - ..qt::config::CustomNoCodeQaConfig::default() + golden_column: "answer".to_owned(), }, }, )); @@ -312,8 +312,8 @@ mod tests { r#" [benchmarks.nocode_resume_test] type = "custom_nocode" -style = "qa" -dataset = "fixture/qa" +style = "exact_match" +dataset = {{ name = "fixture/qa" }} model = "random" prompt_template_file = "{}" golden_column = "answer" diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs index f145435..06fbeb6 100644 --- a/cli/src/commands/run.rs +++ b/cli/src/commands/run.rs @@ -154,49 +154,15 @@ fn assemble_builtin_input( /// `quantiles.toml` `[benchmarks.*]`. #[derive(Serialize)] struct CustomNoCodeConfigInput<'a> { - style: &'a str, - dataset: &'a str, - #[serde(skip_serializing_if = "Option::is_none")] - dataset_config: &'a Option, - #[serde(skip_serializing_if = "Option::is_none")] - split: &'a Option, - #[serde(skip_serializing_if = "Option::is_none")] - revision: &'a Option, + dataset: &'a qt::config::CustomNoCodeDatasetConfig, #[serde(skip_serializing_if = "Option::is_none")] model: &'a Option, - prompt_template_file: &'a str, - #[serde(skip_serializing_if = "Option::is_none")] - golden_column: &'a Option, - #[serde(skip_serializing_if = "Option::is_none")] - golden_index_column: &'a Option, - #[serde(skip_serializing_if = "is_zero")] - golden_index_base: usize, - #[serde(skip_serializing_if = "Option::is_none")] - correct_choice_column: &'a Option, - #[serde(skip_serializing_if = "Option::is_none")] - choices_column: &'a Option, - #[serde(skip_serializing_if = "Option::is_none")] - choice_columns: &'a Option>, - #[serde(skip_serializing_if = "Option::is_none")] - choice_labels: &'a Option>, - #[serde(skip_serializing_if = "is_false")] - shuffle_choices: bool, - #[serde(skip_serializing_if = "Option::is_none")] - shuffle_seed_column: &'a Option, #[serde(skip_serializing_if = "Option::is_none")] limit: Option, #[serde(skip_serializing_if = "Option::is_none")] max_workers: Option, -} - -#[expect(clippy::trivially_copy_pass_by_ref)] -const fn is_zero(value: &usize) -> bool { - *value == 0 -} - -#[expect(clippy::trivially_copy_pass_by_ref)] -const fn is_false(value: &bool) -> bool { - !*value + #[serde(flatten)] + task: &'a qt::config::CustomNoCodeTaskConfig, } pub(super) fn assemble_custom_nocode_input( @@ -208,24 +174,11 @@ pub(super) fn assemble_custom_nocode_input( } let input = CustomNoCodeConfigInput { - style: "qa", dataset: &bench.dataset, - dataset_config: &bench.dataset_config, - split: &bench.split, - revision: &bench.revision, model: &bench.model, - prompt_template_file: &bench.qa.prompt_template_file, - golden_column: &bench.qa.golden_column, - golden_index_column: &bench.qa.golden_index_column, - golden_index_base: bench.qa.golden_index_base, - correct_choice_column: &bench.qa.correct_choice_column, - choices_column: &bench.qa.choices_column, - choice_columns: &bench.qa.choice_columns, - choice_labels: &bench.qa.choice_labels, - shuffle_choices: bench.qa.shuffle_choices, - shuffle_seed_column: &bench.qa.shuffle_seed_column, - limit: bench.qa.limit, - max_workers: bench.qa.max_workers, + limit: bench.limit, + max_workers: bench.max_workers, + task: &bench.task, }; serde_json::to_string(&input).expect("infallible serialization") @@ -797,27 +750,27 @@ mod tests { fn assemble_custom_nocode_input_from_config() { let bench = qt::config::CustomNoCodeBenchmarkConfig { type_: "custom_nocode".to_owned(), - style: qt::config::CustomNoCodeStyle::Qa, - dataset: "quantiles/simpleqa-verified".to_owned(), - dataset_config: Some("default".to_owned()), - split: Some("test".to_owned()), - revision: Some("main".to_owned()), + dataset: qt::config::CustomNoCodeDatasetConfig { + name: "quantiles/simpleqa-verified".to_owned(), + config_name: Some("default".to_owned()), + split: Some("test".to_owned()), + revision: Some("main".to_owned()), + }, model: Some(qt::llm::Sampler::Random {}), - qa: qt::config::CustomNoCodeQaConfig { + limit: Some(10), + max_workers: Some(4), + task: qt::config::CustomNoCodeTaskConfig::ExactMatch { prompt_template_file: "prompts/qa.txt".to_owned(), - golden_column: Some("answer".to_owned()), - limit: Some(10), - max_workers: Some(4), - ..qt::config::CustomNoCodeQaConfig::default() + golden_column: "answer".to_owned(), }, }; let input = super::assemble_custom_nocode_input(&bench, None); let parsed: serde_json::Value = serde_json::from_str(&input).unwrap(); - assert_eq!(parsed["style"], "qa"); - assert_eq!(parsed["dataset"], "quantiles/simpleqa-verified"); - assert_eq!(parsed["dataset_config"], "default"); - assert_eq!(parsed["split"], "test"); - assert_eq!(parsed["revision"], "main"); + assert_eq!(parsed["style"], "exact_match"); + assert_eq!(parsed["dataset"]["name"], "quantiles/simpleqa-verified"); + assert_eq!(parsed["dataset"]["config_name"], "default"); + assert_eq!(parsed["dataset"]["split"], "test"); + assert_eq!(parsed["dataset"]["revision"], "main"); assert_eq!(parsed["model"], "random"); assert_eq!(parsed["prompt_template_file"], "prompts/qa.txt"); assert_eq!(parsed["golden_column"], "answer"); @@ -831,18 +784,18 @@ mod tests { fn assemble_custom_nocode_input_omits_none_fields() { let bench = qt::config::CustomNoCodeBenchmarkConfig { type_: "custom_nocode".to_owned(), - style: qt::config::CustomNoCodeStyle::Qa, - dataset: "quantiles/simpleqa-verified".to_owned(), - dataset_config: None, - split: None, - revision: None, + dataset: qt::config::CustomNoCodeDatasetConfig { + name: "quantiles/simpleqa-verified".to_owned(), + config_name: None, + split: None, + revision: None, + }, model: None, - qa: qt::config::CustomNoCodeQaConfig { + limit: None, + max_workers: None, + task: qt::config::CustomNoCodeTaskConfig::ExactMatch { prompt_template_file: "prompts/qa.txt".to_owned(), - golden_column: Some("answer".to_owned()), - limit: None, - max_workers: None, - ..qt::config::CustomNoCodeQaConfig::default() + golden_column: "answer".to_owned(), }, }; let input = super::assemble_custom_nocode_input(&bench, None); @@ -858,18 +811,18 @@ mod tests { fn assemble_custom_nocode_input_with_cli_override() { let bench = qt::config::CustomNoCodeBenchmarkConfig { type_: "custom_nocode".to_owned(), - style: qt::config::CustomNoCodeStyle::Qa, - dataset: "quantiles/simpleqa-verified".to_owned(), - dataset_config: None, - split: None, - revision: None, + dataset: qt::config::CustomNoCodeDatasetConfig { + name: "quantiles/simpleqa-verified".to_owned(), + config_name: None, + split: None, + revision: None, + }, model: None, - qa: qt::config::CustomNoCodeQaConfig { + limit: None, + max_workers: None, + task: qt::config::CustomNoCodeTaskConfig::ExactMatch { prompt_template_file: "prompts/qa.txt".to_owned(), - golden_column: Some("answer".to_owned()), - limit: None, - max_workers: None, - ..qt::config::CustomNoCodeQaConfig::default() + golden_column: "answer".to_owned(), }, }; let input = super::assemble_custom_nocode_input(&bench, Some(r#"{"limit":5}"#)); diff --git a/cli/src/config/custom_nocode.rs b/cli/src/config/custom_nocode.rs index 4e7c934..28e2ca7 100644 --- a/cli/src/config/custom_nocode.rs +++ b/cli/src/config/custom_nocode.rs @@ -1,186 +1,195 @@ -use std::collections::HashSet; - use anyhow::{Result, bail}; -use serde::{Deserialize, Deserializer}; +use serde::{Deserialize, Serialize}; use crate::llm::Sampler; -/// Style of a no-code custom benchmark. -/// -/// Future variants may include: -/// - `Judge` – evaluate responses against a rubric using a judge model. -/// - `Similarity` – score responses with a similarity metric. -/// - `Agent` – run multi-step agent loops. -#[derive(Debug, Clone)] -pub enum CustomNoCodeStyle { - /// A qa-style benchmark. These benchmarks generally have datasets with - /// a prompt and a golden answer, which the model under test must exactly - /// match (modulo minor whitespace and other cleanup) for it to "pass" - /// the sample - Qa, +/// Task-specific configuration for a no-code benchmark. +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(tag = "style", rename_all = "snake_case")] +pub enum CustomNoCodeTaskConfig { + /// Compare the trimmed model response with a golden dataset column. + ExactMatch { + prompt_template_file: String, + golden_column: String, + }, + /// Render labeled choices and score the selected label. + MultipleChoice { + prompt_template_file: String, + choices: CustomNoCodeChoiceSource, + answer: CustomNoCodeAnswerSource, + choice_labels: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + shuffle: Option, + }, } -impl<'de> Deserialize<'de> for CustomNoCodeStyle { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - match s.as_str() { - "qa" => Ok(CustomNoCodeStyle::Qa), - other => Err(serde::de::Error::custom(format!( - "unsupported custom_nocode style `{other}`; expected `qa`", - ))), - } - } +/// Source of the answer choices for a multiple-choice task. +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(untagged)] +pub enum CustomNoCodeChoiceSource { + Column(CustomNoCodeChoiceColumn), + Columns(CustomNoCodeChoiceColumns), } -/// QA-specific configuration for a no-code custom benchmark. -#[derive(Debug, Deserialize, Clone, Default)] +#[derive(Debug, Deserialize, Serialize, Clone)] #[serde(deny_unknown_fields)] -pub struct CustomNoCodeQaConfig { - /// Path to a Jinja template file for rendering prompts. - pub prompt_template_file: String, - /// Dataset column containing the golden answer label or text. - pub golden_column: Option, - /// Dataset column containing a zero- or one-based golden answer index. - pub golden_index_column: Option, - /// Base used by `golden_index_column`. Defaults to zero. - #[serde(default)] - pub golden_index_base: usize, - /// Dataset column whose value identifies the correct choice before shuffling. - pub correct_choice_column: Option, - /// Dataset column containing choices as an array or object. - pub choices_column: Option, - /// Dataset columns containing choices in their original order. - pub choice_columns: Option>, - /// Labels assigned to choices, for example `["A", "B", "C", "D"]`. - pub choice_labels: Option>, - /// Deterministically shuffle choices before rendering the prompt. - #[serde(default)] - pub shuffle_choices: bool, - /// Dataset column used as the stable seed for choice shuffling. - pub shuffle_seed_column: Option, - /// Number of dataset rows to evaluate. - pub limit: Option, - /// Maximum concurrent workers. - pub max_workers: Option, +pub struct CustomNoCodeChoiceColumn { + pub column: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct CustomNoCodeChoiceColumns { + pub columns: Vec, +} + +/// Source of the correct answer for a multiple-choice task. +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(untagged)] +pub enum CustomNoCodeAnswerSource { + LabelColumn(CustomNoCodeLabelAnswer), + IndexColumn(CustomNoCodeIndexAnswer), + CorrectChoiceColumn(CustomNoCodeCorrectChoiceAnswer), +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct CustomNoCodeLabelAnswer { + pub label_column: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct CustomNoCodeIndexAnswer { + pub index_column: String, + #[serde(default, skip_serializing_if = "is_zero")] + pub index_base: usize, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct CustomNoCodeCorrectChoiceAnswer { + pub correct_choice_column: String, +} + +/// Deterministic choice-shuffling configuration. +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct CustomNoCodeShuffleConfig { + pub seed_column: String, } /// No-code custom benchmark configuration. #[derive(Debug, Deserialize, Clone)] -#[serde(deny_unknown_fields)] pub struct CustomNoCodeBenchmarkConfig { #[serde(rename = "type")] pub type_: String, - pub style: CustomNoCodeStyle, - /// Dataset identifier in `HuggingFace` (e.g. `quantiles/simpleqa-verified`). - pub dataset: String, + /// Hugging Face dataset coordinates. + pub dataset: CustomNoCodeDatasetConfig, + /// Model sampler to use. + pub model: Option, + /// Number of dataset rows to evaluate. + pub limit: Option, + /// Maximum concurrent workers. + pub max_workers: Option, + #[serde(flatten)] + pub task: CustomNoCodeTaskConfig, +} + +/// Hugging Face dataset coordinates for a no-code benchmark. +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct CustomNoCodeDatasetConfig { + /// Dataset identifier, for example `quantiles/simpleqa-verified`. + pub name: String, /// Optional Hugging Face dataset configuration/subset. - pub dataset_config: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub config_name: Option, /// Optional dataset split. The dataset manager chooses one when omitted. + #[serde(skip_serializing_if = "Option::is_none")] pub split: Option, /// Optional dataset revision. + #[serde(skip_serializing_if = "Option::is_none")] pub revision: Option, - /// Model sampler to use. - pub model: Option, - #[serde(flatten)] - pub qa: CustomNoCodeQaConfig, } -impl CustomNoCodeQaConfig { - /// Validate mutually exclusive answer and choice source fields. - pub(crate) fn validate(&self) -> Result<()> { - let answer_sources = [ - self.golden_column.is_some(), - self.golden_index_column.is_some(), - self.correct_choice_column.is_some(), - ] - .into_iter() - .filter(|configured| *configured) - .count(); - if answer_sources != 1 { - bail!( - "custom_nocode QA config requires exactly one of `golden_column`, `golden_index_column`, or `correct_choice_column`" - ); +impl CustomNoCodeTaskConfig { + #[must_use] + pub fn prompt_template_file(&self) -> &str { + match self { + Self::ExactMatch { + prompt_template_file, + .. + } + | Self::MultipleChoice { + prompt_template_file, + .. + } => prompt_template_file, } + } - let choice_sources = [self.choices_column.is_some(), self.choice_columns.is_some()] - .into_iter() - .filter(|configured| *configured) - .count(); - if choice_sources > 1 { - bail!( - "custom_nocode QA config accepts only one of `choices_column` or `choice_columns`" - ); - } + pub(crate) fn validate(&self) -> Result<()> { + let Self::MultipleChoice { + choices, + answer, + choice_labels, + .. + } = self + else { + return Ok(()); + }; - let is_multiple_choice = choice_sources == 1; - if (self.golden_index_column.is_some() || self.correct_choice_column.is_some()) - && !is_multiple_choice - { - bail!( - "`golden_index_column` and `correct_choice_column` require a multiple-choice source" - ); - } - if is_multiple_choice && self.choice_labels.as_ref().is_none_or(Vec::is_empty) { - bail!("multiple-choice custom_nocode QA config requires non-empty `choice_labels`"); - } - if !is_multiple_choice && self.choice_labels.is_some() { - bail!("`choice_labels` requires `choices_column` or `choice_columns`"); - } - if self.shuffle_choices && !is_multiple_choice { - bail!("`shuffle_choices` requires `choices_column` or `choice_columns`"); + if choice_labels.is_empty() { + bail!("multiple-choice `choice_labels` must not be empty"); } - if self.shuffle_choices && self.shuffle_seed_column.is_none() { - bail!("`shuffle_choices = true` requires `shuffle_seed_column`"); - } - if !self.shuffle_choices && self.shuffle_seed_column.is_some() { - bail!("`shuffle_seed_column` requires `shuffle_choices = true`"); - } - if self.golden_index_column.is_none() && self.golden_index_base != 0 { - bail!("`golden_index_base` requires `golden_index_column`"); + let unique: std::collections::HashSet<&str> = + choice_labels.iter().map(String::as_str).collect(); + if unique.len() != choice_labels.len() { + bail!("multiple-choice `choice_labels` must contain unique values"); } - if let Some(columns) = &self.choice_columns { + if let CustomNoCodeChoiceSource::Columns(CustomNoCodeChoiceColumns { columns }) = choices { if columns.is_empty() { - bail!("`choice_columns` must not be empty"); + bail!("multiple-choice `choices.columns` must not be empty"); } - if let Some(labels) = &self.choice_labels - && labels.len() != columns.len() - { - bail!("`choice_labels` and `choice_columns` must have the same length"); + if choice_labels.len() != columns.len() { + bail!("`choice_labels` and `choices.columns` must have the same length"); } - if let Some(correct) = &self.correct_choice_column - && !columns.contains(correct) + if let CustomNoCodeAnswerSource::CorrectChoiceColumn(CustomNoCodeCorrectChoiceAnswer { + correct_choice_column, + }) = answer + && !columns.contains(correct_choice_column) { - bail!("`correct_choice_column` must be present in `choice_columns`"); - } - } - - if let Some(labels) = &self.choice_labels { - let unique: HashSet<&str> = labels.iter().map(String::as_str).collect(); - if unique.len() != labels.len() { - bail!("`choice_labels` must contain unique values"); + bail!("`answer.correct_choice_column` must be present in `choices.columns`"); } + } else if matches!(answer, CustomNoCodeAnswerSource::CorrectChoiceColumn(_)) { + bail!("`answer.correct_choice_column` requires `choices.columns`"); } Ok(()) } } +#[expect( + clippy::trivially_copy_pass_by_ref, + reason = "serde skip_serializing_if predicates receive references" +)] +const fn is_zero(value: &usize) -> bool { + *value == 0 +} + #[cfg(test)] mod tests { use super::super::{BenchmarkConfig, WorkspaceConfig}; use super::*; #[test] - fn deserialize_custom_nocode_qa() { + fn deserialize_custom_nocode_exact_match() { let toml = r#" [benchmarks.nocode_custom] type = "custom_nocode" - style = "qa" - dataset = "quantiles/simpleqa-verified" + style = "exact_match" + dataset = { name = "quantiles/simpleqa-verified" } model = "random" prompt_template_file = "prompts/qa.txt" golden_column = "answer" @@ -190,12 +199,18 @@ mod tests { let bench = config.benchmarks.get("nocode_custom").unwrap(); assert!(matches!(bench, BenchmarkConfig::CustomNoCode(_))); if let BenchmarkConfig::CustomNoCode(c) = bench { - assert!(matches!(c.style, CustomNoCodeStyle::Qa)); - assert_eq!(c.dataset, "quantiles/simpleqa-verified"); + assert_eq!(c.dataset.name, "quantiles/simpleqa-verified"); assert_eq!(c.model, Some(Sampler::Random)); - assert_eq!(c.qa.prompt_template_file, "prompts/qa.txt"); - assert_eq!(c.qa.golden_column.as_deref(), Some("answer")); - assert_eq!(c.qa.limit, Some(10)); + assert_eq!(c.limit, Some(10)); + let CustomNoCodeTaskConfig::ExactMatch { + prompt_template_file, + golden_column, + } = &c.task + else { + panic!("expected exact-match task"); + }; + assert_eq!(prompt_template_file, "prompts/qa.txt"); + assert_eq!(golden_column, "answer"); } } @@ -205,7 +220,7 @@ mod tests { [benchmarks.nocode_custom] type = "custom_nocode" style = "judge" - dataset = "quantiles/simpleqa-verified" + dataset = { name = "quantiles/simpleqa-verified" } model = "random" prompt_template_file = "prompts/qa.txt" golden_column = "answer" @@ -219,8 +234,8 @@ mod tests { let toml = r#" [benchmarks.nocode_custom] type = "custom_nocode" - style = "qa" - dataset = "quantiles/simpleqa-verified" + style = "exact_match" + dataset = { name = "quantiles/simpleqa-verified" } model = "random" golden_column = "answer" "#; @@ -232,18 +247,18 @@ mod tests { fn validate_rejects_missing_template_file() { let bench = BenchmarkConfig::CustomNoCode(Box::new(CustomNoCodeBenchmarkConfig { type_: "custom_nocode".to_owned(), - style: CustomNoCodeStyle::Qa, - dataset: "quantiles/simpleqa-verified".to_owned(), - dataset_config: None, - split: None, - revision: None, + dataset: CustomNoCodeDatasetConfig { + name: "quantiles/simpleqa-verified".to_owned(), + config_name: None, + split: None, + revision: None, + }, model: Some(Sampler::Random), - qa: CustomNoCodeQaConfig { + limit: None, + max_workers: None, + task: CustomNoCodeTaskConfig::ExactMatch { prompt_template_file: "does_not_exist.txt".to_owned(), - golden_column: Some("answer".to_owned()), - limit: None, - max_workers: None, - ..CustomNoCodeQaConfig::default() + golden_column: "answer".to_owned(), }, })); let err = bench.validate().unwrap_err(); @@ -255,18 +270,18 @@ mod tests { let file = tempfile::NamedTempFile::new().unwrap(); let bench = BenchmarkConfig::CustomNoCode(Box::new(CustomNoCodeBenchmarkConfig { type_: "custom_nocode".to_owned(), - style: CustomNoCodeStyle::Qa, - dataset: "quantiles/simpleqa-verified".to_owned(), - dataset_config: None, - split: None, - revision: None, + dataset: CustomNoCodeDatasetConfig { + name: "quantiles/simpleqa-verified".to_owned(), + config_name: None, + split: None, + revision: None, + }, model: Some(Sampler::Random), - qa: CustomNoCodeQaConfig { + limit: None, + max_workers: None, + task: CustomNoCodeTaskConfig::ExactMatch { prompt_template_file: file.path().to_str().unwrap().to_owned(), - golden_column: Some("answer".to_owned()), - limit: None, - max_workers: None, - ..CustomNoCodeQaConfig::default() + golden_column: "answer".to_owned(), }, })); bench.validate().unwrap(); @@ -288,5 +303,32 @@ mod tests { "missing custom_nocode example `{name}`" ); } + + let Some(BenchmarkConfig::CustomNoCode(medmcqa)) = config.benchmarks.get("medmcqa") else { + panic!("missing MedMCQA example"); + }; + assert!(matches!( + medmcqa.task, + CustomNoCodeTaskConfig::MultipleChoice { .. } + )); + assert_eq!(medmcqa.dataset.name, "quantiles/medmcqa"); + assert_eq!(medmcqa.dataset.split.as_deref(), Some("validation")); + } + + #[test] + fn multiple_choice_rejects_mixed_answer_sources() { + let toml = r#" + [benchmarks.invalid] + type = "custom_nocode" + style = "multiple_choice" + dataset = { name = "fixture/qa" } + prompt_template_file = "prompts/qa.txt" + choices = { column = "options" } + answer = { label_column = "answer", index_column = "answer_index" } + choice_labels = ["A", "B"] + "#; + + let result: Result = toml::from_str(toml); + assert!(result.is_err()); } } diff --git a/cli/src/config/mod.rs b/cli/src/config/mod.rs index 86c32df..f6e9117 100644 --- a/cli/src/config/mod.rs +++ b/cli/src/config/mod.rs @@ -32,13 +32,13 @@ impl BenchmarkConfig { Ok(()) } BenchmarkConfig::CustomNoCode(c) => { - if !std::path::Path::new(&c.qa.prompt_template_file).is_file() { + if !std::path::Path::new(c.task.prompt_template_file()).is_file() { bail!( "custom_nocode benchmark config `prompt_template_file` must point to an existing file. File `{}` was not found", - c.qa.prompt_template_file + c.task.prompt_template_file() ); } - c.qa.validate()?; + c.task.validate()?; Ok(()) } } diff --git a/custom-nocode-examples/quantiles.toml b/custom-nocode-examples/quantiles.toml index 5879207..3ad7eac 100644 --- a/custom-nocode-examples/quantiles.toml +++ b/custom-nocode-examples/quantiles.toml @@ -1,7 +1,7 @@ [benchmarks.nocode_custom] type = "custom_nocode" -style = "qa" -dataset = "quantiles/simpleqa-verified" +style = "exact_match" +dataset = { name = "quantiles/simpleqa-verified" } model = "random" prompt_template_file = "prompts/qa.txt" golden_column = "answer" @@ -9,55 +9,45 @@ limit = 10 [benchmarks.medqa] type = "custom_nocode" -style = "qa" -dataset = "GBaker/MedQA-USMLE-4-options" -dataset_config = "default" -split = "test" +style = "multiple_choice" +dataset = { name = "quantiles/MedQA-USMLE-4-options", config_name = "default", split = "test" } model = "random" prompt_template_file = "prompts/medqa.txt" -choices_column = "options" +choices = { column = "options" } choice_labels = ["A", "B", "C", "D"] -golden_column = "answer_idx" +answer = { label_column = "answer_idx" } limit = 10 [benchmarks.medmcqa] type = "custom_nocode" -style = "qa" -dataset = "openlifescienceai/medmcqa" -dataset_config = "default" -split = "test" +style = "multiple_choice" +dataset = { name = "quantiles/medmcqa", config_name = "default", split = "validation" } model = "random" prompt_template_file = "prompts/medmcqa.txt" -choice_columns = ["opa", "opb", "opc", "opd"] +choices = { columns = ["opa", "opb", "opc", "opd"] } choice_labels = ["A", "B", "C", "D"] -golden_index_column = "cop" -golden_index_base = 0 +answer = { index_column = "cop", index_base = 0 } limit = 10 [benchmarks.mmlu-pro] type = "custom_nocode" -style = "qa" -dataset = "TIGER-Lab/MMLU-Pro" -dataset_config = "default" -split = "test" +style = "multiple_choice" +dataset = { name = "TIGER-Lab/MMLU-Pro", config_name = "default", split = "test" } model = "random" prompt_template_file = "prompts/mmlu-pro.txt" -choices_column = "options" +choices = { column = "options" } choice_labels = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"] -golden_column = "answer" +answer = { label_column = "answer" } limit = 10 [benchmarks.gpqa] type = "custom_nocode" -style = "qa" -dataset = "Wanfq/gpqa" -dataset_config = "gpqa_diamond" -split = "train" +style = "multiple_choice" +dataset = { name = "Wanfq/gpqa", config_name = "gpqa_diamond", split = "train" } model = "random" prompt_template_file = "prompts/gpqa.txt" -choice_columns = ["Correct Answer", "Incorrect Answer 1", "Incorrect Answer 2", "Incorrect Answer 3"] +choices = { columns = ["Correct Answer", "Incorrect Answer 1", "Incorrect Answer 2", "Incorrect Answer 3"] } choice_labels = ["A", "B", "C", "D"] -correct_choice_column = "Correct Answer" -shuffle_choices = true -shuffle_seed_column = "Record ID" +answer = { correct_choice_column = "Correct Answer" } +shuffle = { seed_column = "Record ID" } limit = 10 From 7fc4424235ec7ab8548fe7657e70810cbca373bc Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:50:50 -0700 Subject: [PATCH 03/16] progress --- CONFIG.md | 42 ++++++++---- cli/README.md | 5 +- cli/src/builtins/custom_nocode.rs | 45 ++++++------ cli/src/commands/resume.rs | 7 +- cli/src/commands/run.rs | 25 +++---- cli/src/config/custom_nocode.rs | 98 ++++++++++++++------------- cli/src/config/mod.rs | 6 +- custom-nocode-examples/quantiles.toml | 26 ++----- 8 files changed, 126 insertions(+), 128 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index d71a524..6cd432c 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -105,11 +105,10 @@ No-code benchmarks are configured in TOML and run natively inside the CLI, witho ```toml [benchmarks.nocode_custom] type = "custom_nocode" -style = "exact_match" +style = { type = "exact_match", golden_column = "answer" } dataset = { name = "quantiles/simpleqa-verified" } model = "random" prompt_template_file = "prompts/qa.txt" -golden_column = "answer" limit = 10 ``` @@ -124,7 +123,8 @@ For a complete minimal example, see [`custom-nocode-examples/quantiles.toml`](./ | Field | Type | Required | Description | |--|--|--|--| | `type` | string | yes | Must be `"custom_nocode"`. | -| `style` | string | yes | `"exact_match"` for open-answer or label exact match, or `"multiple_choice"` for choice-based benchmarks. | +| `style` | table | yes | Scoring style and its style-specific configuration. | +| `style.type` | string | yes | `"exact_match"` for open-answer or label exact match, or `"multiple_choice"` for choice-based benchmarks. | | `dataset` | table | yes | Hugging Face dataset coordinates. | | `dataset.name` | string | yes | Dataset identifier, for example `"quantiles/simpleqa-verified"`. | | `dataset.config_name` | string | no | Hugging Face dataset configuration or subset. | @@ -132,21 +132,33 @@ For a complete minimal example, see [`custom-nocode-examples/quantiles.toml`](./ | `dataset.revision` | string | no | Dataset revision. | | `model` | string or table | no | Model sampler. Defaults to the demo random sampler. See [model naming](#model-naming). | | `prompt_template_file` | string | yes | Path to a Jinja prompt template file. The template receives the complete dataset `row` and, for multiple-choice benchmarks, normalized `choices`. | -| `golden_column` | string | conditional | Dataset column containing the golden answer. Required for `exact_match`. | -| `choices` | table | conditional | Choice source. Required for `multiple_choice`. Configure exactly one of `choices.column` or `choices.columns`. | -| `choices.column` | string | conditional | Dataset column containing choices as an array or label-keyed object. | -| `choices.columns` | array of strings | conditional | Dataset columns containing choices in their original order. | -| `answer` | table | conditional | Correct-answer source. Required for `multiple_choice`. Configure exactly one answer-source form. | -| `answer.label_column` | string | conditional | Dataset column containing the golden choice label. | -| `answer.index_column` | string | conditional | Dataset column containing the golden choice index. | -| `answer.index_base` | integer | no | Index base for `answer.index_column`. Defaults to `0`. | -| `answer.correct_choice_column` | string | conditional | Member of `choices.columns` known to contain the correct answer. | -| `choice_labels` | array of strings | conditional | Labels assigned to choices in order. Required for multiple choice; array-backed rows may use a prefix of the configured labels. | -| `shuffle` | table | no | Enables deterministic choice shuffling for `multiple_choice`. | -| `shuffle.seed_column` | string | conditional | Stable row identifier used to seed deterministic shuffling. Required when `shuffle` is present. | +| `style.golden_column` | string | conditional | Dataset column containing the golden answer. Required for `exact_match`. | +| `style.choices` | table | conditional | Choice source. Required for `multiple_choice`. Configure exactly one of `style.choices.column` or `style.choices.columns`. | +| `style.choices.column` | string | conditional | Dataset column containing choices as an array or label-keyed object. | +| `style.choices.columns` | array of strings | conditional | Dataset columns containing choices in their original order. | +| `style.answer` | table | conditional | Correct-answer source. Required for `multiple_choice`. Configure exactly one answer-source form. | +| `style.answer.label_column` | string | conditional | Dataset column containing the golden choice label. | +| `style.answer.index_column` | string | conditional | Dataset column containing the golden choice index. | +| `style.answer.index_base` | integer | no | Index base for `style.answer.index_column`. Defaults to `0`. | +| `style.answer.correct_choice_column` | string | conditional | Member of `style.choices.columns` known to contain the correct answer. | +| `style.choice_labels` | array of strings | conditional | Labels assigned to choices in order. Required for multiple choice; array-backed rows may use a prefix of the configured labels. | +| `style.shuffle` | table | no | Enables deterministic choice shuffling for `multiple_choice`. | +| `style.shuffle.seed_column` | string | conditional | Stable row identifier used to seed deterministic shuffling. Required when `style.shuffle` is present. | | `limit` | integer | no | Number of dataset rows to evaluate. | | `max_workers` | integer | no | Maximum concurrent workers. | +Multiple-choice configuration keeps its choice and answer sources inside `style`: + +```toml +[benchmarks.medqa] +type = "custom_nocode" +style = { type = "multiple_choice", choices = { column = "options" }, choice_labels = ["A", "B", "C", "D"], answer = { label_column = "answer_idx" } } +dataset = { name = "quantiles/MedQA-USMLE-4-options", config_name = "default", split = "test" } +model = "random" +prompt_template_file = "prompts/medqa.txt" +limit = 10 +``` + Templates access dataset fields directly. A multiple-choice template can iterate the normalized choices: ```jinja diff --git a/cli/README.md b/cli/README.md index 9dadecf..3d0285e 100644 --- a/cli/README.md +++ b/cli/README.md @@ -73,16 +73,15 @@ max_workers = 100 ### No-code QA benchmarks -For dataset-backed QA checks that do not need custom evaluation code, set `type = "custom_nocode"`. Use `style = "exact_match"` for benchmarks that test an exact match to a golden answer, and `style = "multiple_choice"` for choice-based benchmarks. The benchmark runs inside the CLI, renders each prompt with the configured Jinja template, calls the configured model, and scores each row against the configured answer source. +For dataset-backed QA checks that do not need custom evaluation code, set `type = "custom_nocode"`. Set `style.type` to `"exact_match"` for benchmarks that test an exact match to a golden answer, or `"multiple_choice"` for choice-based benchmarks. Style-specific dataset columns belong inside the `style` table. The benchmark runs inside the CLI, renders each prompt with the configured Jinja template, calls the configured model, and scores each row against the configured answer source. ```toml [benchmarks.nocode_custom] type = "custom_nocode" -style = "exact_match" +style = { type = "exact_match", golden_column = "answer" } dataset = { name = "quantiles/simpleqa-verified" } model = "random" prompt_template_file = "prompts/qa.txt" -golden_column = "answer" limit = 10 ``` diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index 44bda97..6ab32c0 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -21,8 +21,8 @@ struct CustomNoCodeInput { model: Option, limit: Option, max_workers: Option, - #[serde(flatten)] - task: crate::config::CustomNoCodeTaskConfig, + prompt_template_file: String, + style: crate::config::CustomNoCodeStyleConfig, } /// Per-row step output stored as JSON in the step record. @@ -54,7 +54,7 @@ struct EvaluateRowArgs<'a> { /// The row value row: &'a serde_json::Value, /// Exact-match or multiple-choice task configuration. - task: &'a crate::config::CustomNoCodeTaskConfig, + style: &'a crate::config::CustomNoCodeStyleConfig, /// The prompt template template_str: &'a str, /// The pre-constructed jinja template env @@ -84,7 +84,7 @@ impl CustomNoCodeBuiltin { } async fn evaluate_row(&self, args: EvaluateRowArgs<'_>) -> Result { - let prepared = prepare_row(args.i, args.row, args.task)?; + let prepared = prepare_row(args.i, args.row, args.style)?; let rendered = args .env @@ -116,9 +116,10 @@ impl CustomNoCodeBuiltin { .with_context(|| format!("failed to sample LLM for row {}", args.i))?; let parsed_response = if prepared.is_multiple_choice { - let crate::config::CustomNoCodeTaskConfig::MultipleChoice { - choice_labels, .. - } = args.task + let crate::config::CustomNoCodeStyleConfig::MultipleChoice { + choice_labels, + .. + } = args.style else { unreachable!("prepared multiple-choice row has exact-match config") }; @@ -163,7 +164,7 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { async fn execute(&self, ctx: BuiltinContext<'_>) -> Result<()> { let config = parse_input(ctx.input)?; - let (template_str, env) = load_template(config.task.prompt_template_file())?; + let (template_str, env) = load_template(&config.prompt_template_file)?; let max_workers = config.max_workers.unwrap_or_else(get_max_workers); let llm = resolve_sampler(config.model.as_ref(), || Arc::new(RandomSampler::new(80)))?; @@ -191,7 +192,7 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { .map_or("random".to_string(), std::string::ToString::to_string); let run_id = ctx.run_id; let dataset = config.dataset.name.clone(); - let task = Arc::new(config.task.clone()); + let style = Arc::new(config.style.clone()); let template_str = Arc::new(template_str); let name = self.name(); @@ -203,13 +204,13 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { let db = db.clone(); let model_name = model_name.clone(); let template_str = Arc::clone(&template_str); - let task = Arc::clone(&task); + let style = Arc::clone(&style); let env = env.clone(); async move { let args = EvaluateRowArgs { i, row: &row, - task: &task, + style: &style, template_str: &template_str, env: &env, model_name: &model_name, @@ -281,17 +282,17 @@ async fn resolve_dataset_limit( fn prepare_row( row_index: usize, row: &serde_json::Value, - task: &crate::config::CustomNoCodeTaskConfig, + style: &crate::config::CustomNoCodeStyleConfig, ) -> Result { - let crate::config::CustomNoCodeTaskConfig::MultipleChoice { + let crate::config::CustomNoCodeStyleConfig::MultipleChoice { choices: choice_source, answer, choice_labels, shuffle, .. - } = task + } = style else { - let crate::config::CustomNoCodeTaskConfig::ExactMatch { golden_column, .. } = task else { + let crate::config::CustomNoCodeStyleConfig::ExactMatch { golden_column } = style else { unreachable!() }; let golden = extract_scalar(row, golden_column) @@ -581,9 +582,8 @@ mod tests { fn multiple_choice_config( choices: crate::config::CustomNoCodeChoiceSource, answer: crate::config::CustomNoCodeAnswerSource, - ) -> crate::config::CustomNoCodeTaskConfig { - crate::config::CustomNoCodeTaskConfig::MultipleChoice { - prompt_template_file: "unused.txt".to_owned(), + ) -> crate::config::CustomNoCodeStyleConfig { + crate::config::CustomNoCodeStyleConfig::MultipleChoice { choices, answer, choice_labels: ["A", "B", "C", "D"].map(str::to_owned).to_vec(), @@ -660,8 +660,7 @@ mod tests { #[test] fn prepares_gpqa_with_deterministic_shuffle() { - let config = crate::config::CustomNoCodeTaskConfig::MultipleChoice { - prompt_template_file: "unused.txt".to_owned(), + let config = crate::config::CustomNoCodeStyleConfig::MultipleChoice { choices: crate::config::CustomNoCodeChoiceSource::Columns( crate::config::CustomNoCodeChoiceColumns { columns: [ @@ -719,10 +718,9 @@ mod tests { std::fs::write(&template_path, "{{ unclosed").unwrap(); let input_json = serde_json::to_string(&json!({ - "style": "exact_match", + "style": {"type": "exact_match", "golden_column": "a"}, "dataset": {"name": "fixture/qa"}, "prompt_template_file": template_path.to_str().unwrap(), - "golden_column": "a", })) .unwrap(); @@ -807,11 +805,10 @@ mod tests { // Assemble the input JSON that execute() expects. let input_json = serde_json::to_string(&json!({ - "style": "exact_match", + "style": {"type": "exact_match", "golden_column": "answer"}, "dataset": {"name": "fixture/qa"}, "model": "random", "prompt_template_file": template_path.to_str().unwrap(), - "golden_column": "answer", "limit": 2, })) .unwrap(); diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs index bf0de50..deddf41 100644 --- a/cli/src/commands/resume.rs +++ b/cli/src/commands/resume.rs @@ -235,10 +235,10 @@ mod tests { revision: None, }, model: Some(qt::llm::Sampler::Random {}), + prompt_template_file: file.path().to_str().unwrap().to_owned(), limit: None, max_workers: None, - task: qt::config::CustomNoCodeTaskConfig::ExactMatch { - prompt_template_file: file.path().to_str().unwrap().to_owned(), + style: qt::config::CustomNoCodeStyleConfig::ExactMatch { golden_column: "answer".to_owned(), }, }, @@ -312,11 +312,10 @@ mod tests { r#" [benchmarks.nocode_resume_test] type = "custom_nocode" -style = "exact_match" +style = {{ type = "exact_match", golden_column = "answer" }} dataset = {{ name = "fixture/qa" }} model = "random" prompt_template_file = "{}" -golden_column = "answer" limit = 2 "#, template_path.to_str().unwrap() diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs index 06fbeb6..e9a1ca6 100644 --- a/cli/src/commands/run.rs +++ b/cli/src/commands/run.rs @@ -161,8 +161,8 @@ struct CustomNoCodeConfigInput<'a> { limit: Option, #[serde(skip_serializing_if = "Option::is_none")] max_workers: Option, - #[serde(flatten)] - task: &'a qt::config::CustomNoCodeTaskConfig, + prompt_template_file: &'a str, + style: &'a qt::config::CustomNoCodeStyleConfig, } pub(super) fn assemble_custom_nocode_input( @@ -178,7 +178,8 @@ pub(super) fn assemble_custom_nocode_input( model: &bench.model, limit: bench.limit, max_workers: bench.max_workers, - task: &bench.task, + prompt_template_file: &bench.prompt_template_file, + style: &bench.style, }; serde_json::to_string(&input).expect("infallible serialization") @@ -316,7 +317,7 @@ pub async fn execute_builtin(args: ExecuteBuiltinArgs<'_>) -> Result<()> { } else { print_aggregate_metrics_table(&aggregate); println!( - "\nRun `qt show {} --verbose` for sample-level details.", + "\nRun `qt show {} --json` for sample-level details.", args.run_id ); } @@ -757,23 +758,23 @@ mod tests { revision: Some("main".to_owned()), }, model: Some(qt::llm::Sampler::Random {}), + prompt_template_file: "prompts/qa.txt".to_owned(), limit: Some(10), max_workers: Some(4), - task: qt::config::CustomNoCodeTaskConfig::ExactMatch { - prompt_template_file: "prompts/qa.txt".to_owned(), + style: qt::config::CustomNoCodeStyleConfig::ExactMatch { golden_column: "answer".to_owned(), }, }; let input = super::assemble_custom_nocode_input(&bench, None); let parsed: serde_json::Value = serde_json::from_str(&input).unwrap(); - assert_eq!(parsed["style"], "exact_match"); + assert_eq!(parsed["style"]["type"], "exact_match"); assert_eq!(parsed["dataset"]["name"], "quantiles/simpleqa-verified"); assert_eq!(parsed["dataset"]["config_name"], "default"); assert_eq!(parsed["dataset"]["split"], "test"); assert_eq!(parsed["dataset"]["revision"], "main"); assert_eq!(parsed["model"], "random"); assert_eq!(parsed["prompt_template_file"], "prompts/qa.txt"); - assert_eq!(parsed["golden_column"], "answer"); + assert_eq!(parsed["style"]["golden_column"], "answer"); assert_eq!(parsed["limit"], 10); assert_eq!(parsed["max_workers"], 4); } @@ -791,10 +792,10 @@ mod tests { revision: None, }, model: None, + prompt_template_file: "prompts/qa.txt".to_owned(), limit: None, max_workers: None, - task: qt::config::CustomNoCodeTaskConfig::ExactMatch { - prompt_template_file: "prompts/qa.txt".to_owned(), + style: qt::config::CustomNoCodeStyleConfig::ExactMatch { golden_column: "answer".to_owned(), }, }; @@ -818,10 +819,10 @@ mod tests { revision: None, }, model: None, + prompt_template_file: "prompts/qa.txt".to_owned(), limit: None, max_workers: None, - task: qt::config::CustomNoCodeTaskConfig::ExactMatch { - prompt_template_file: "prompts/qa.txt".to_owned(), + style: qt::config::CustomNoCodeStyleConfig::ExactMatch { golden_column: "answer".to_owned(), }, }; diff --git a/cli/src/config/custom_nocode.rs b/cli/src/config/custom_nocode.rs index 28e2ca7..63161be 100644 --- a/cli/src/config/custom_nocode.rs +++ b/cli/src/config/custom_nocode.rs @@ -3,18 +3,14 @@ use serde::{Deserialize, Serialize}; use crate::llm::Sampler; -/// Task-specific configuration for a no-code benchmark. +/// Scoring style and style-specific configuration for a no-code benchmark. #[derive(Debug, Deserialize, Serialize, Clone)] -#[serde(tag = "style", rename_all = "snake_case")] -pub enum CustomNoCodeTaskConfig { +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum CustomNoCodeStyleConfig { /// Compare the trimmed model response with a golden dataset column. - ExactMatch { - prompt_template_file: String, - golden_column: String, - }, + ExactMatch { golden_column: String }, /// Render labeled choices and score the selected label. MultipleChoice { - prompt_template_file: String, choices: CustomNoCodeChoiceSource, answer: CustomNoCodeAnswerSource, choice_labels: Vec, @@ -81,6 +77,7 @@ pub struct CustomNoCodeShuffleConfig { /// No-code custom benchmark configuration. #[derive(Debug, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub struct CustomNoCodeBenchmarkConfig { #[serde(rename = "type")] pub type_: String, @@ -88,12 +85,14 @@ pub struct CustomNoCodeBenchmarkConfig { pub dataset: CustomNoCodeDatasetConfig, /// Model sampler to use. pub model: Option, + /// Path to a Jinja template file for rendering prompts. + pub prompt_template_file: String, /// Number of dataset rows to evaluate. pub limit: Option, /// Maximum concurrent workers. pub max_workers: Option, - #[serde(flatten)] - pub task: CustomNoCodeTaskConfig, + /// Scoring style and its required dataset-column configuration. + pub style: CustomNoCodeStyleConfig, } /// Hugging Face dataset coordinates for a no-code benchmark. @@ -113,21 +112,7 @@ pub struct CustomNoCodeDatasetConfig { pub revision: Option, } -impl CustomNoCodeTaskConfig { - #[must_use] - pub fn prompt_template_file(&self) -> &str { - match self { - Self::ExactMatch { - prompt_template_file, - .. - } - | Self::MultipleChoice { - prompt_template_file, - .. - } => prompt_template_file, - } - } - +impl CustomNoCodeStyleConfig { pub(crate) fn validate(&self) -> Result<()> { let Self::MultipleChoice { choices, @@ -188,11 +173,10 @@ mod tests { let toml = r#" [benchmarks.nocode_custom] type = "custom_nocode" - style = "exact_match" + style = { type = "exact_match", golden_column = "answer" } dataset = { name = "quantiles/simpleqa-verified" } model = "random" prompt_template_file = "prompts/qa.txt" - golden_column = "answer" limit = 10 "#; let config: WorkspaceConfig = toml::from_str(toml).unwrap(); @@ -202,14 +186,10 @@ mod tests { assert_eq!(c.dataset.name, "quantiles/simpleqa-verified"); assert_eq!(c.model, Some(Sampler::Random)); assert_eq!(c.limit, Some(10)); - let CustomNoCodeTaskConfig::ExactMatch { - prompt_template_file, - golden_column, - } = &c.task - else { + let CustomNoCodeStyleConfig::ExactMatch { golden_column } = &c.style else { panic!("expected exact-match task"); }; - assert_eq!(prompt_template_file, "prompts/qa.txt"); + assert_eq!(c.prompt_template_file, "prompts/qa.txt"); assert_eq!(golden_column, "answer"); } } @@ -219,11 +199,10 @@ mod tests { let toml = r#" [benchmarks.nocode_custom] type = "custom_nocode" - style = "judge" + style = { type = "judge", golden_column = "answer" } dataset = { name = "quantiles/simpleqa-verified" } model = "random" prompt_template_file = "prompts/qa.txt" - golden_column = "answer" "#; let result: Result = toml::from_str(toml); assert!(result.is_err()); @@ -234,10 +213,9 @@ mod tests { let toml = r#" [benchmarks.nocode_custom] type = "custom_nocode" - style = "exact_match" + style = { type = "exact_match", golden_column = "answer" } dataset = { name = "quantiles/simpleqa-verified" } model = "random" - golden_column = "answer" "#; let result: Result = toml::from_str(toml); assert!(result.is_err()); @@ -254,10 +232,10 @@ mod tests { revision: None, }, model: Some(Sampler::Random), + prompt_template_file: "does_not_exist.txt".to_owned(), limit: None, max_workers: None, - task: CustomNoCodeTaskConfig::ExactMatch { - prompt_template_file: "does_not_exist.txt".to_owned(), + style: CustomNoCodeStyleConfig::ExactMatch { golden_column: "answer".to_owned(), }, })); @@ -277,10 +255,10 @@ mod tests { revision: None, }, model: Some(Sampler::Random), + prompt_template_file: file.path().to_str().unwrap().to_owned(), limit: None, max_workers: None, - task: CustomNoCodeTaskConfig::ExactMatch { - prompt_template_file: file.path().to_str().unwrap().to_owned(), + style: CustomNoCodeStyleConfig::ExactMatch { golden_column: "answer".to_owned(), }, })); @@ -308,8 +286,8 @@ mod tests { panic!("missing MedMCQA example"); }; assert!(matches!( - medmcqa.task, - CustomNoCodeTaskConfig::MultipleChoice { .. } + medmcqa.style, + CustomNoCodeStyleConfig::MultipleChoice { .. } )); assert_eq!(medmcqa.dataset.name, "quantiles/medmcqa"); assert_eq!(medmcqa.dataset.split.as_deref(), Some("validation")); @@ -320,12 +298,38 @@ mod tests { let toml = r#" [benchmarks.invalid] type = "custom_nocode" - style = "multiple_choice" + style = { type = "multiple_choice", choices = { column = "options" }, answer = { label_column = "answer", index_column = "answer_index" }, choice_labels = ["A", "B"] } + dataset = { name = "fixture/qa" } + prompt_template_file = "prompts/qa.txt" + "#; + + let result: Result = toml::from_str(toml); + assert!(result.is_err()); + } + + #[test] + fn exact_match_rejects_multiple_choice_fields() { + let toml = r#" + [benchmarks.invalid] + type = "custom_nocode" + style = { type = "exact_match", golden_column = "answer", choices = { column = "options" } } dataset = { name = "fixture/qa" } prompt_template_file = "prompts/qa.txt" - choices = { column = "options" } - answer = { label_column = "answer", index_column = "answer_index" } - choice_labels = ["A", "B"] + "#; + + let result: Result = toml::from_str(toml); + assert!(result.is_err()); + } + + #[test] + fn benchmark_rejects_style_fields_at_top_level() { + let toml = r#" + [benchmarks.invalid] + type = "custom_nocode" + style = { type = "exact_match", golden_column = "answer" } + dataset = { name = "fixture/qa" } + prompt_template_file = "prompts/qa.txt" + golden_column = "answer" "#; let result: Result = toml::from_str(toml); diff --git a/cli/src/config/mod.rs b/cli/src/config/mod.rs index f6e9117..fdbcabc 100644 --- a/cli/src/config/mod.rs +++ b/cli/src/config/mod.rs @@ -32,13 +32,13 @@ impl BenchmarkConfig { Ok(()) } BenchmarkConfig::CustomNoCode(c) => { - if !std::path::Path::new(c.task.prompt_template_file()).is_file() { + if !std::path::Path::new(&c.prompt_template_file).is_file() { bail!( "custom_nocode benchmark config `prompt_template_file` must point to an existing file. File `{}` was not found", - c.task.prompt_template_file() + c.prompt_template_file ); } - c.task.validate()?; + c.style.validate()?; Ok(()) } } diff --git a/custom-nocode-examples/quantiles.toml b/custom-nocode-examples/quantiles.toml index 3ad7eac..ab022ba 100644 --- a/custom-nocode-examples/quantiles.toml +++ b/custom-nocode-examples/quantiles.toml @@ -1,53 +1,39 @@ [benchmarks.nocode_custom] type = "custom_nocode" -style = "exact_match" +style = { type = "exact_match", golden_column = "answer" } dataset = { name = "quantiles/simpleqa-verified" } model = "random" prompt_template_file = "prompts/qa.txt" -golden_column = "answer" limit = 10 [benchmarks.medqa] type = "custom_nocode" -style = "multiple_choice" +style = { type = "multiple_choice", choices = { column = "options" }, choice_labels = ["A", "B", "C", "D"], answer = { label_column = "answer_idx" } } dataset = { name = "quantiles/MedQA-USMLE-4-options", config_name = "default", split = "test" } model = "random" prompt_template_file = "prompts/medqa.txt" -choices = { column = "options" } -choice_labels = ["A", "B", "C", "D"] -answer = { label_column = "answer_idx" } limit = 10 [benchmarks.medmcqa] type = "custom_nocode" -style = "multiple_choice" +style = { type = "multiple_choice", choices = { columns = ["opa", "opb", "opc", "opd"] }, choice_labels = ["A", "B", "C", "D"], answer = { index_column = "cop", index_base = 0 } } dataset = { name = "quantiles/medmcqa", config_name = "default", split = "validation" } model = "random" prompt_template_file = "prompts/medmcqa.txt" -choices = { columns = ["opa", "opb", "opc", "opd"] } -choice_labels = ["A", "B", "C", "D"] -answer = { index_column = "cop", index_base = 0 } -limit = 10 +limit = 100 [benchmarks.mmlu-pro] type = "custom_nocode" -style = "multiple_choice" +style = { type = "multiple_choice", choices = { column = "options" }, choice_labels = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"], answer = { label_column = "answer" } } dataset = { name = "TIGER-Lab/MMLU-Pro", config_name = "default", split = "test" } model = "random" prompt_template_file = "prompts/mmlu-pro.txt" -choices = { column = "options" } -choice_labels = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"] -answer = { label_column = "answer" } limit = 10 [benchmarks.gpqa] type = "custom_nocode" -style = "multiple_choice" +style = { type = "multiple_choice", choices = { columns = ["Correct Answer", "Incorrect Answer 1", "Incorrect Answer 2", "Incorrect Answer 3"] }, choice_labels = ["A", "B", "C", "D"], answer = { correct_choice_column = "Correct Answer" }, shuffle = { seed_column = "Record ID" } } dataset = { name = "Wanfq/gpqa", config_name = "gpqa_diamond", split = "train" } model = "random" prompt_template_file = "prompts/gpqa.txt" -choices = { columns = ["Correct Answer", "Incorrect Answer 1", "Incorrect Answer 2", "Incorrect Answer 3"] } -choice_labels = ["A", "B", "C", "D"] -answer = { correct_choice_column = "Correct Answer" } -shuffle = { seed_column = "Record ID" } limit = 10 From ebfae3d5388f210eea39f1e85b8b3b71bf9713e2 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:53:42 -0700 Subject: [PATCH 04/16] upping MMLU-pro limit --- custom-nocode-examples/quantiles.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom-nocode-examples/quantiles.toml b/custom-nocode-examples/quantiles.toml index ab022ba..4a95ec0 100644 --- a/custom-nocode-examples/quantiles.toml +++ b/custom-nocode-examples/quantiles.toml @@ -28,7 +28,7 @@ style = { type = "multiple_choice", choices = { column = "options" }, choice_lab dataset = { name = "TIGER-Lab/MMLU-Pro", config_name = "default", split = "test" } model = "random" prompt_template_file = "prompts/mmlu-pro.txt" -limit = 10 +limit = 1000 [benchmarks.gpqa] type = "custom_nocode" From 228ad5ce1cd636ae407f6a803f2dad6e8897bc76 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:56:06 -0700 Subject: [PATCH 05/16] using quantiles datasets --- custom-nocode-examples/quantiles.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/custom-nocode-examples/quantiles.toml b/custom-nocode-examples/quantiles.toml index 4a95ec0..33349c4 100644 --- a/custom-nocode-examples/quantiles.toml +++ b/custom-nocode-examples/quantiles.toml @@ -25,7 +25,7 @@ limit = 100 [benchmarks.mmlu-pro] type = "custom_nocode" style = { type = "multiple_choice", choices = { column = "options" }, choice_labels = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"], answer = { label_column = "answer" } } -dataset = { name = "TIGER-Lab/MMLU-Pro", config_name = "default", split = "test" } +dataset = { name = "quantiles/MMLU-Pro", config_name = "default", split = "test" } model = "random" prompt_template_file = "prompts/mmlu-pro.txt" limit = 1000 @@ -33,7 +33,6 @@ limit = 1000 [benchmarks.gpqa] type = "custom_nocode" style = { type = "multiple_choice", choices = { columns = ["Correct Answer", "Incorrect Answer 1", "Incorrect Answer 2", "Incorrect Answer 3"] }, choice_labels = ["A", "B", "C", "D"], answer = { correct_choice_column = "Correct Answer" }, shuffle = { seed_column = "Record ID" } } -dataset = { name = "Wanfq/gpqa", config_name = "gpqa_diamond", split = "train" } +dataset = { name = "quantiles/gpqa", config_name = "gpqa_diamond", split = "train" } model = "random" prompt_template_file = "prompts/gpqa.txt" -limit = 10 From fccfc6f20d3458faf759d4e8eaeac7ff437274a8 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:57:16 -0700 Subject: [PATCH 06/16] removing unnecessary `unreachable!(...)` macro usage --- cli/src/builtins/custom_nocode.rs | 343 ++++++++++++++---------------- 1 file changed, 165 insertions(+), 178 deletions(-) diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index 6ab32c0..7f14d10 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -44,7 +44,6 @@ struct PromptChoice { struct PreparedRow { choices: Vec, golden: String, - is_multiple_choice: bool, } /// Arguments for the [`CustomNoCodeBuiltin::evaluate_row`] method @@ -115,17 +114,14 @@ impl CustomNoCodeBuiltin { .await .with_context(|| format!("failed to sample LLM for row {}", args.i))?; - let parsed_response = if prepared.is_multiple_choice { - let crate::config::CustomNoCodeStyleConfig::MultipleChoice { + let parsed_response = match args.style { + crate::config::CustomNoCodeStyleConfig::MultipleChoice { choice_labels, .. - } = args.style - else { - unreachable!("prepared multiple-choice row has exact-match config") - }; - extract_choice_label(&model_response, choice_labels) - } else { - Some(model_response.trim().to_owned()) + } => extract_choice_label(&model_response, choice_labels), + crate::config::CustomNoCodeStyleConfig::ExactMatch { .. } => { + Some(model_response.trim().to_owned()) + } }; let is_correct = parsed_response.as_deref() == Some(prepared.golden.trim()); @@ -284,80 +280,77 @@ fn prepare_row( row: &serde_json::Value, style: &crate::config::CustomNoCodeStyleConfig, ) -> Result { - let crate::config::CustomNoCodeStyleConfig::MultipleChoice { - choices: choice_source, - answer, - choice_labels, - shuffle, - .. - } = style - else { - let crate::config::CustomNoCodeStyleConfig::ExactMatch { golden_column } = style else { - unreachable!() - }; - let golden = extract_scalar(row, golden_column) - .with_context(|| format!("row {row_index}: missing golden column `{golden_column}`"))?; - return Ok(PreparedRow { - choices: Vec::new(), - golden, - is_multiple_choice: false, - }); - }; - - let choice_values = extract_choices(row_index, row, choice_source, choice_labels)?; - if choice_values.is_empty() || choice_values.len() > choice_labels.len() { - bail!( - "row {row_index}: found {} choices but configured only {} labels", - choice_values.len(), - choice_labels.len() - ); - } - let labels = &choice_labels[..choice_values.len()]; - - let correct_index = resolve_correct_index( - row_index, - row, - choice_source, - answer, - labels, - &choice_values, - )?; - let mut indexed_choices: Vec<(String, bool)> = choice_values - .into_iter() - .enumerate() - .map(|(index, text)| (text, index == correct_index)) - .collect(); - - if let Some(shuffle) = shuffle { - let seed = extract_scalar(row, &shuffle.seed_column).with_context(|| { - format!( - "row {row_index}: missing shuffle seed column `{}`", - shuffle.seed_column - ) - })?; - deterministic_shuffle(&mut indexed_choices, &seed); - } - - let mut golden = None; - let choices = indexed_choices - .into_iter() - .zip(labels) - .map(|((text, is_correct), label)| { - if is_correct { - golden = Some(label.clone()); + match style { + crate::config::CustomNoCodeStyleConfig::ExactMatch { golden_column } => { + let golden = extract_scalar(row, golden_column).with_context(|| { + format!("row {row_index}: missing golden column `{golden_column}`") + })?; + Ok(PreparedRow { + choices: Vec::new(), + golden, + }) + } + crate::config::CustomNoCodeStyleConfig::MultipleChoice { + choices: choice_source, + answer, + choice_labels, + shuffle, + } => { + let choice_values = extract_choices(row_index, row, choice_source, choice_labels)?; + if choice_values.is_empty() || choice_values.len() > choice_labels.len() { + bail!( + "row {row_index}: found {} choices but configured only {} labels", + choice_values.len(), + choice_labels.len() + ); } - PromptChoice { - label: label.clone(), - text, + let labels = &choice_labels[..choice_values.len()]; + + let correct_index = resolve_correct_index( + row_index, + row, + choice_source, + answer, + labels, + &choice_values, + )?; + let mut indexed_choices: Vec<(String, bool)> = choice_values + .into_iter() + .enumerate() + .map(|(index, text)| (text, index == correct_index)) + .collect(); + + if let Some(shuffle) = shuffle { + let seed = extract_scalar(row, &shuffle.seed_column).with_context(|| { + format!( + "row {row_index}: missing shuffle seed column `{}`", + shuffle.seed_column + ) + })?; + deterministic_shuffle(&mut indexed_choices, &seed); } - }) - .collect(); - - Ok(PreparedRow { - choices, - golden: golden.context("correct choice disappeared while assigning labels")?, - is_multiple_choice: true, - }) + + let mut golden = None; + let choices = indexed_choices + .into_iter() + .zip(labels) + .map(|((text, is_correct), label)| { + if is_correct { + golden = Some(label.clone()); + } + PromptChoice { + label: label.clone(), + text, + } + }) + .collect(); + + Ok(PreparedRow { + choices, + golden: golden.context("correct choice disappeared while assigning labels")?, + }) + } + } } fn extract_choices( @@ -366,51 +359,47 @@ fn extract_choices( source: &crate::config::CustomNoCodeChoiceSource, labels: &[String], ) -> Result> { - if let crate::config::CustomNoCodeChoiceSource::Columns( - crate::config::CustomNoCodeChoiceColumns { columns }, - ) = source - { - return columns + match source { + crate::config::CustomNoCodeChoiceSource::Columns( + crate::config::CustomNoCodeChoiceColumns { columns }, + ) => columns .iter() .map(|column| { extract_scalar(row, column) .with_context(|| format!("row {row_index}: missing choice column `{column}`")) }) - .collect(); - } - - let crate::config::CustomNoCodeChoiceSource::Column(crate::config::CustomNoCodeChoiceColumn { - column, - }) = source - else { - unreachable!() - }; - let value = row - .get(column) - .with_context(|| format!("row {row_index}: missing choices column `{column}`"))?; - let owned; - let value = if let Some(encoded) = value.as_str() { - owned = serde_json::from_str(encoded).unwrap_or_else(|_| value.clone()); - &owned - } else { - value - }; - - match value { - serde_json::Value::Array(values) => values - .iter() - .map(value_to_scalar) - .collect::>>() - .with_context(|| format!("row {row_index}: choices column `{column}` contains non-scalar values")), - serde_json::Value::Object(values) => labels - .iter() - .map(|label| { - values.get(label).and_then(value_to_scalar).with_context(|| { - format!("row {row_index}: choices object `{column}` has no scalar `{label}` value") - }) - }) .collect(), - _ => bail!("row {row_index}: choices column `{column}` must be an array or object"), + crate::config::CustomNoCodeChoiceSource::Column( + crate::config::CustomNoCodeChoiceColumn { column }, + ) => { + let value = row + .get(column) + .with_context(|| format!("row {row_index}: missing choices column `{column}`"))?; + let owned; + let value = if let Some(encoded) = value.as_str() { + owned = serde_json::from_str(encoded).unwrap_or_else(|_| value.clone()); + &owned + } else { + value + }; + + match value { + serde_json::Value::Array(values) => values + .iter() + .map(value_to_scalar) + .collect::>>() + .with_context(|| format!("row {row_index}: choices column `{column}` contains non-scalar values")), + serde_json::Value::Object(values) => labels + .iter() + .map(|label| { + values.get(label).and_then(value_to_scalar).with_context(|| { + format!("row {row_index}: choices object `{column}` has no scalar `{label}` value") + }) + }) + .collect(), + _ => bail!("row {row_index}: choices column `{column}` must be an array or object"), + } + } } } @@ -422,66 +411,64 @@ fn resolve_correct_index( labels: &[String], choices: &[String], ) -> Result { - if let crate::config::CustomNoCodeAnswerSource::IndexColumn( - crate::config::CustomNoCodeIndexAnswer { - index_column: column, - index_base, - }, - ) = answer - { - let raw = row - .get(column) - .with_context(|| format!("row {row_index}: missing golden index column `{column}`"))?; - let index = raw - .as_u64() - .or_else(|| raw.as_str().and_then(|value| value.parse().ok())) - .with_context(|| { - format!("row {row_index}: golden index column `{column}` is not an integer") + match answer { + crate::config::CustomNoCodeAnswerSource::IndexColumn( + crate::config::CustomNoCodeIndexAnswer { + index_column: column, + index_base, + }, + ) => { + let raw = row.get(column).with_context(|| { + format!("row {row_index}: missing golden index column `{column}`") })?; - let index = usize::try_from(index) - .ok() - .and_then(|index| index.checked_sub(*index_base)) - .with_context(|| format!("row {row_index}: golden index is below configured base"))?; - if index >= choices.len() { - bail!("row {row_index}: golden index {index} is outside the choice list"); + let index = raw + .as_u64() + .or_else(|| raw.as_str().and_then(|value| value.parse().ok())) + .with_context(|| { + format!("row {row_index}: golden index column `{column}` is not an integer") + })?; + let index = usize::try_from(index) + .ok() + .and_then(|index| index.checked_sub(*index_base)) + .with_context(|| { + format!("row {row_index}: golden index is below configured base") + })?; + if index >= choices.len() { + bail!("row {row_index}: golden index {index} is outside the choice list"); + } + Ok(index) + } + crate::config::CustomNoCodeAnswerSource::CorrectChoiceColumn( + crate::config::CustomNoCodeCorrectChoiceAnswer { + correct_choice_column: column, + }, + ) => { + let crate::config::CustomNoCodeChoiceSource::Columns( + crate::config::CustomNoCodeChoiceColumns { columns }, + ) = choice_source + else { + bail!("correct-choice answer requires column-backed choices"); + }; + columns + .iter() + .position(|candidate| candidate == column) + .context("validated correct choice is absent from choice columns") + } + crate::config::CustomNoCodeAnswerSource::LabelColumn( + crate::config::CustomNoCodeLabelAnswer { + label_column: column, + }, + ) => { + let golden = extract_scalar(row, column) + .with_context(|| format!("row {row_index}: missing golden column `{column}`"))?; + labels + .iter() + .position(|label| label.eq_ignore_ascii_case(golden.trim())) + .with_context(|| { + format!("row {row_index}: golden label `{golden}` is not in `choice_labels`") + }) } - return Ok(index); - } - - if let crate::config::CustomNoCodeAnswerSource::CorrectChoiceColumn( - crate::config::CustomNoCodeCorrectChoiceAnswer { - correct_choice_column: column, - }, - ) = answer - { - let crate::config::CustomNoCodeChoiceSource::Columns( - crate::config::CustomNoCodeChoiceColumns { columns }, - ) = choice_source - else { - bail!("correct-choice answer requires column-backed choices"); - }; - return columns - .iter() - .position(|candidate| candidate == column) - .context("validated correct choice is absent from choice columns"); } - - let crate::config::CustomNoCodeAnswerSource::LabelColumn( - crate::config::CustomNoCodeLabelAnswer { - label_column: column, - }, - ) = answer - else { - unreachable!() - }; - let golden = extract_scalar(row, column) - .with_context(|| format!("row {row_index}: missing golden column `{column}`"))?; - labels - .iter() - .position(|label| label.eq_ignore_ascii_case(golden.trim())) - .with_context(|| { - format!("row {row_index}: golden label `{golden}` is not in `choice_labels`") - }) } fn extract_scalar(row: &serde_json::Value, key: &str) -> Option { From a432d84823355cfadc536b25408726584401eb9e Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:05:45 -0700 Subject: [PATCH 07/16] better config formatting --- custom-nocode-examples/quantiles.toml | 36 +++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/custom-nocode-examples/quantiles.toml b/custom-nocode-examples/quantiles.toml index 33349c4..28b04cd 100644 --- a/custom-nocode-examples/quantiles.toml +++ b/custom-nocode-examples/quantiles.toml @@ -8,31 +8,57 @@ limit = 10 [benchmarks.medqa] type = "custom_nocode" -style = { type = "multiple_choice", choices = { column = "options" }, choice_labels = ["A", "B", "C", "D"], answer = { label_column = "answer_idx" } } dataset = { name = "quantiles/MedQA-USMLE-4-options", config_name = "default", split = "test" } model = "random" prompt_template_file = "prompts/medqa.txt" limit = 10 + [benchmarks.medqa.style] + type = "multiple_choice" + choices = { column = "options" } + choice_labels = ["A", "B", "C", "D"] + answer = { label_column = "answer_idx" } + + [benchmarks.medmcqa] type = "custom_nocode" -style = { type = "multiple_choice", choices = { columns = ["opa", "opb", "opc", "opd"] }, choice_labels = ["A", "B", "C", "D"], answer = { index_column = "cop", index_base = 0 } } dataset = { name = "quantiles/medmcqa", config_name = "default", split = "validation" } model = "random" prompt_template_file = "prompts/medmcqa.txt" limit = 100 + [benchmarks.medmcqa.style] + type = "multiple_choice" + choices = { columns = ["opa", "opb", "opc", "opd"] } + choice_labels = ["A", "B", "C", "D"] + answer = { index_column = "cop", index_base = 0 } + [benchmarks.mmlu-pro] type = "custom_nocode" -style = { type = "multiple_choice", choices = { column = "options" }, choice_labels = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"], answer = { label_column = "answer" } } dataset = { name = "quantiles/MMLU-Pro", config_name = "default", split = "test" } model = "random" prompt_template_file = "prompts/mmlu-pro.txt" limit = 1000 + [benchmarks.mmlu-pro.style] + type = "multiple_choice" + choices = { column = "options" } + choice_labels = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"] + answer = { label_column = "answer" } + [benchmarks.gpqa] type = "custom_nocode" -style = { type = "multiple_choice", choices = { columns = ["Correct Answer", "Incorrect Answer 1", "Incorrect Answer 2", "Incorrect Answer 3"] }, choice_labels = ["A", "B", "C", "D"], answer = { correct_choice_column = "Correct Answer" }, shuffle = { seed_column = "Record ID" } } -dataset = { name = "quantiles/gpqa", config_name = "gpqa_diamond", split = "train" } model = "random" prompt_template_file = "prompts/gpqa.txt" + + [benchmarks.gpqa.style] + type = "multiple_choice" + choices = { columns = ["Correct Answer", "Incorrect Answer 1", "Incorrect Answer 2", "Incorrect Answer 3"] } + choice_labels = ["A", "B", "C", "D"] + answer = { correct_choice_column = "Correct Answer" } + shuffle = { seed_column = "Record ID" } + + [benchmarks.gpqa.dataset] + name = "quantiles/gpqa" + config_name = "gpqa_diamond" + split = "train" From 1fa177adadc0444e3d0b2873e0244df1e3f8f8c6 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:10:10 -0700 Subject: [PATCH 08/16] progress --- custom-nocode-examples/quantiles.toml | 6 +----- mise.toml | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/custom-nocode-examples/quantiles.toml b/custom-nocode-examples/quantiles.toml index 28b04cd..b2f2293 100644 --- a/custom-nocode-examples/quantiles.toml +++ b/custom-nocode-examples/quantiles.toml @@ -50,6 +50,7 @@ limit = 1000 type = "custom_nocode" model = "random" prompt_template_file = "prompts/gpqa.txt" +dataset = { name = "quantiles/gpqa", config_name = "gpqa_diamond", split = "train" } [benchmarks.gpqa.style] type = "multiple_choice" @@ -57,8 +58,3 @@ prompt_template_file = "prompts/gpqa.txt" choice_labels = ["A", "B", "C", "D"] answer = { correct_choice_column = "Correct Answer" } shuffle = { seed_column = "Record ID" } - - [benchmarks.gpqa.dataset] - name = "quantiles/gpqa" - config_name = "gpqa_diamond" - split = "train" diff --git a/mise.toml b/mise.toml index 4fd48a6..7f2b742 100644 --- a/mise.toml +++ b/mise.toml @@ -10,3 +10,23 @@ bun = "1.3.14" rust = "1.96.0" # keep this version inline with the version declared in .github/workflows/python uv = "0.11.21" + +[tasks.run-custom-nocode] +description = "Run all configured no-code benchmarks with the random model" +dir = "custom-nocode-examples" +run = ''' +aggregate_exit=0 + +for benchmark in nocode_custom medqa medmcqa mmlu-pro gpqa; do + printf '\n===== %s =====\n' "$benchmark" + if qt run "$benchmark"; then + printf '===== %s: PASS =====\n' "$benchmark" + else + exit_code=$? + printf '===== %s: FAIL (exit %s) =====\n' "$benchmark" "$exit_code" + aggregate_exit=1 + fi +done + +exit "$aggregate_exit" +''' From 316845bfc3b9f2906c8ed87a6b6bead23e9bc337 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:11:11 -0700 Subject: [PATCH 09/16] progress --- mise.toml | 18 +----------------- scripts/run-custom-nocode.sh | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 17 deletions(-) create mode 100755 scripts/run-custom-nocode.sh diff --git a/mise.toml b/mise.toml index 7f2b742..c966c9e 100644 --- a/mise.toml +++ b/mise.toml @@ -13,20 +13,4 @@ uv = "0.11.21" [tasks.run-custom-nocode] description = "Run all configured no-code benchmarks with the random model" -dir = "custom-nocode-examples" -run = ''' -aggregate_exit=0 - -for benchmark in nocode_custom medqa medmcqa mmlu-pro gpqa; do - printf '\n===== %s =====\n' "$benchmark" - if qt run "$benchmark"; then - printf '===== %s: PASS =====\n' "$benchmark" - else - exit_code=$? - printf '===== %s: FAIL (exit %s) =====\n' "$benchmark" "$exit_code" - aggregate_exit=1 - fi -done - -exit "$aggregate_exit" -''' +run = "./scripts/run-custom-nocode.sh" diff --git a/scripts/run-custom-nocode.sh b/scripts/run-custom-nocode.sh new file mode 100755 index 0000000..9998060 --- /dev/null +++ b/scripts/run-custom-nocode.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -u + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root/custom-nocode-examples" || exit 1 + +aggregate_exit=0 + +for benchmark in nocode_custom medqa medmcqa mmlu-pro gpqa; do + printf '\n===== %s =====\n' "$benchmark" + if qt run "$benchmark"; then + printf '===== %s: PASS =====\n' "$benchmark" + else + exit_code=$? + printf '===== %s: FAIL (exit %s) =====\n' "$benchmark" "$exit_code" + aggregate_exit=1 + fi +done + +exit "$aggregate_exit" From 2358bbe3712119eb36db7bba2993a45e2f3502ad Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:19:26 -0700 Subject: [PATCH 10/16] progress --- CONFIG.md | 2 + cli/README.md | 2 + cli/src/builtins/custom_nocode.rs | 61 ++++++++++++++++++++++++++++++- cli/src/llm/random_label.rs | 7 +++- 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 6cd432c..b5f8766 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -102,6 +102,8 @@ Note that models require specific configuration based on the provider. For detai No-code benchmarks are configured in TOML and run natively inside the CLI, without a custom Python or TypeScript evaluation program. The `exact_match` style scores an open answer or label against a golden answer column. The `multiple_choice` style normalizes choices, extracts the selected label from the response, and scores it against a configured label, index, or correct-choice column. +With `model = "random"`, exact-match benchmarks use the demo random-text sampler, while multiple-choice benchmarks uniformly sample one of the configured `style.choice_labels`. + ```toml [benchmarks.nocode_custom] type = "custom_nocode" diff --git a/cli/README.md b/cli/README.md index 3d0285e..7471590 100644 --- a/cli/README.md +++ b/cli/README.md @@ -75,6 +75,8 @@ max_workers = 100 For dataset-backed QA checks that do not need custom evaluation code, set `type = "custom_nocode"`. Set `style.type` to `"exact_match"` for benchmarks that test an exact match to a golden answer, or `"multiple_choice"` for choice-based benchmarks. Style-specific dataset columns belong inside the `style` table. The benchmark runs inside the CLI, renders each prompt with the configured Jinja template, calls the configured model, and scores each row against the configured answer source. +With `model = "random"`, exact-match benchmarks generate demo random text and multiple-choice benchmarks uniformly select from `style.choice_labels`. + ```toml [benchmarks.nocode_custom] type = "custom_nocode" diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index 7f14d10..648ee41 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -12,6 +12,7 @@ use crate::builtins::output::set_builtin_run_output; use crate::builtins::{BuiltinContext, BuiltinWorkflow}; use crate::dataset::DatasetManager; use crate::llm::random::RandomSampler; +use crate::llm::random_label::RandomLabelSampler; /// Input deserialized from the JSON assembled by `commands::run`. #[derive(Debug, Deserialize)] @@ -162,7 +163,7 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { let config = parse_input(ctx.input)?; let (template_str, env) = load_template(&config.prompt_template_file)?; let max_workers = config.max_workers.unwrap_or_else(get_max_workers); - let llm = resolve_sampler(config.model.as_ref(), || Arc::new(RandomSampler::new(80)))?; + let llm = resolve_custom_nocode_sampler(config.model.as_ref(), &config.style)?; let (manager, info, limit) = resolve_dataset_limit( &config.dataset.name, @@ -229,6 +230,19 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { } } +fn resolve_custom_nocode_sampler( + model: Option<&crate::llm::Sampler>, + style: &crate::config::CustomNoCodeStyleConfig, +) -> Result> { + if model.is_none_or(|sampler| matches!(sampler, crate::llm::Sampler::Random)) + && let crate::config::CustomNoCodeStyleConfig::MultipleChoice { choice_labels, .. } = style + { + return Ok(Arc::new(RandomLabelSampler::new(choice_labels))); + } + + resolve_sampler(model, || Arc::new(RandomSampler::new(80))) +} + fn parse_input(input: Option<&str>) -> Result { let config: CustomNoCodeInput = input .map(serde_json::from_str) @@ -566,6 +580,51 @@ mod tests { assert_eq!(extract_choice_label("unknown", &labels), None); } + #[tokio::test] + async fn multiple_choice_random_sampler_uses_configured_labels() { + let style = multiple_choice_config( + crate::config::CustomNoCodeChoiceSource::Column( + crate::config::CustomNoCodeChoiceColumn { + column: "options".to_owned(), + }, + ), + crate::config::CustomNoCodeAnswerSource::LabelColumn( + crate::config::CustomNoCodeLabelAnswer { + label_column: "answer".to_owned(), + }, + ), + ); + let configured_random = crate::llm::Sampler::Random; + + for model in [None, Some(&configured_random)] { + let sampler = resolve_custom_nocode_sampler(model, &style).unwrap(); + for _ in 0..100 { + let response = sampler.sample("ignored prompt").await.unwrap(); + assert!(matches!(response.as_str(), "A" | "B" | "C" | "D")); + } + } + } + + #[tokio::test] + async fn exact_match_random_sampler_remains_alphanumeric() { + let style = crate::config::CustomNoCodeStyleConfig::ExactMatch { + golden_column: "answer".to_owned(), + }; + let sampler = + resolve_custom_nocode_sampler(Some(&crate::llm::Sampler::Random), &style).unwrap(); + + for _ in 0..100 { + let response = sampler.sample("ignored prompt").await.unwrap(); + assert!(!response.is_empty()); + assert!(response.len() <= 80); + assert!( + response + .chars() + .all(|character| character.is_ascii_alphanumeric()) + ); + } + } + fn multiple_choice_config( choices: crate::config::CustomNoCodeChoiceSource, answer: crate::config::CustomNoCodeAnswerSource, diff --git a/cli/src/llm/random_label.rs b/cli/src/llm/random_label.rs index db2ae61..2a976d3 100644 --- a/cli/src/llm/random_label.rs +++ b/cli/src/llm/random_label.rs @@ -17,9 +17,12 @@ pub struct RandomLabelSampler { impl RandomLabelSampler { #[must_use] - pub(crate) fn new(labels: &[&str]) -> Self { + pub(crate) fn new>(labels: &[S]) -> Self { Self { - labels: labels.iter().map(|&s| s.to_string()).collect(), + labels: labels + .iter() + .map(|label| label.as_ref().to_owned()) + .collect(), } } } From 661cb0f291ad205b48a05ba5feea09ff3cfa1d16 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:23:42 -0700 Subject: [PATCH 11/16] progress --- cli/src/dataset/cache.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cli/src/dataset/cache.rs b/cli/src/dataset/cache.rs index 4c6c3e8..d3ec78a 100644 --- a/cli/src/dataset/cache.rs +++ b/cli/src/dataset/cache.rs @@ -11,7 +11,9 @@ use std::fs::File; use std::path::Path; use std::sync::Arc; +/// Arrow field metadata key marking values that were serialized as JSON strings. const JSON_ENCODED_METADATA_KEY: &str = "quantiles.json_encoded"; +/// Version included in cache keys so incompatible on-disk formats use separate entries. const CACHE_FORMAT_VERSION: &str = "v2"; /// Manages the on-disk cache for dataset batches. From 80edde05d9dad06bc603a7ff87b3d83bf640939b Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:25:06 -0700 Subject: [PATCH 12/16] progress --- scripts/run-custom-nocode.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-custom-nocode.sh b/scripts/run-custom-nocode.sh index 9998060..cec65dc 100755 --- a/scripts/run-custom-nocode.sh +++ b/scripts/run-custom-nocode.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -u +set -eou pipefail repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root/custom-nocode-examples" || exit 1 From 5cebd63fc005c8b5bdeb85e156e4737323c6e92c Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:25:45 -0700 Subject: [PATCH 13/16] progress --- CONFIG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONFIG.md b/CONFIG.md index b5f8766..f0861f1 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -8,7 +8,7 @@ You need a config file when you want to do one or more of the following: - Override built-in benchmark defaults (e.g. model, sample limit) - Define custom evaluations (`type = "custom_code"`) -- Define no-code QA benchmarks (`type = "custom_nocode"`) +- Define custom benchmarks without writing or maintaining code (`type = "custom_nocode"`) - Resume custom evaluations later with `qt resume ` You don't, however, need any configuration when you want to run built-in benchmarks. `qt run pubmedqa`, for example, works out of the box. From b5bc9d67df4421da4021511b983e08a47b34d8f5 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:26:46 -0700 Subject: [PATCH 14/16] progress --- CONFIG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index f0861f1..e945bb4 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -7,8 +7,8 @@ Quantiles uses a single TOML configuration file in the current working directory You need a config file when you want to do one or more of the following: - Override built-in benchmark defaults (e.g. model, sample limit) -- Define custom evaluations (`type = "custom_code"`) -- Define custom benchmarks without writing or maintaining code (`type = "custom_nocode"`) +- Build custom evaluations without writing or maintaining code (`type = "custom_nocode"` in the `quantiles.toml` configuration file) +- Build custom evaluations with code (`type = "custom_code"` in the `quantiles.toml` configuration file) - Resume custom evaluations later with `qt resume ` You don't, however, need any configuration when you want to run built-in benchmarks. `qt run pubmedqa`, for example, works out of the box. From e5858eec431c652890697a3aebee343aca2756da Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:32:26 -0700 Subject: [PATCH 15/16] docs --- CONFIG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index e945bb4..252417a 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -100,9 +100,9 @@ Note that models require specific configuration based on the provider. For detai ### `custom_nocode` -No-code benchmarks are configured in TOML and run natively inside the CLI, without a custom Python or TypeScript evaluation program. The `exact_match` style scores an open answer or label against a golden answer column. The `multiple_choice` style normalizes choices, extracts the selected label from the response, and scores it against a configured label, index, or correct-choice column. +No-code evals are configured in TOML and run natively inside the CLI, without any custom Python code. These evals are defined with `type = "custom_nocode"` and a `style` parameter. The `style = "exact_match"` configuration creates an eval that scores an open answer or label against a golden answer column. The `style = "multiple_choice"` configuration normalizes choices, extracts the selected label from the response, and scores it against a configured label, index, or correct-choice column. -With `model = "random"`, exact-match benchmarks use the demo random-text sampler, while multiple-choice benchmarks uniformly sample one of the configured `style.choice_labels`. +When you use a `"random"` model (`model = "random"`) with an `exact_match` eval, your eval will use the built-in "model" that generates random text, so you'll likely to get very low accuracy numbers. When you configure a `multiple_choice` eval with the `"random"` model, the built-in "model" will uniformly sample from one of the the configured `style.choice_labels`, so you can expect higher accuracies than with `exact_match`. In both cases, this model is intended for testing your benchmark. We do not recommend relying on it for any real ML evaluation work. ```toml [benchmarks.nocode_custom] From fbae91428ebe19eda333f804b81af43840e145d9 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:41:57 -0700 Subject: [PATCH 16/16] adding some more sample-level and aggregate metrics --- CONFIG.md | 8 ++ cli/README.md | 2 + cli/src/builtins/custom_nocode.rs | 186 +++++++++++++++++++++++++++++- cli/src/metrics_store.rs | 53 +++++++++ 4 files changed, 245 insertions(+), 4 deletions(-) diff --git a/CONFIG.md b/CONFIG.md index 252417a..64ff395 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -149,6 +149,14 @@ For a complete minimal example, see [`custom-nocode-examples/quantiles.toml`](./ | `limit` | integer | no | Number of dataset rows to evaluate. | | `max_workers` | integer | no | Maximum concurrent workers. | +Each sample emits `is_correct`, `response_parsed`, and `latency_ms`. For exact-match benchmarks, every response is considered parsed; for multiple-choice benchmarks, `response_parsed` is `0` when the response cannot be parsed as a configured choice label. + +Each run emits these aggregate metrics: + +- `accuracy`, `correct_count`, `incorrect_count`, and `total_count` +- `parsed_response_count`, `unparsed_response_count`, and `parse_rate` +- `mean_latency_ms`, `median_latency_ms`, `p95_latency_ms`, `p99_latency_ms`, `min_latency_ms`, and `max_latency_ms` + Multiple-choice configuration keeps its choice and answer sources inside `style`: ```toml diff --git a/cli/README.md b/cli/README.md index 7471590..062df0c 100644 --- a/cli/README.md +++ b/cli/README.md @@ -77,6 +77,8 @@ For dataset-backed QA checks that do not need custom evaluation code, set `type With `model = "random"`, exact-match benchmarks generate demo random text and multiple-choice benchmarks uniformly select from `style.choice_labels`. +Samples emit correctness, response-parsing, and latency metrics. Runs aggregate accuracy and counts, parse success, and mean, median, p95, p99, minimum, and maximum latency. + ```toml [benchmarks.nocode_custom] type = "custom_nocode" diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index 648ee41..a8435d5 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -4,7 +4,8 @@ use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; use crate::builtins::common::{ - emit_accuracy_metrics, get_max_workers, hash_input, resolve_sampler, run_timed_step, + compute_statistics, emit_accuracy_metrics, get_max_workers, hash_input, resolve_sampler, + run_timed_step, }; use crate::builtins::dataset_runner::DatasetRunner; use crate::builtins::input::set_builtin_run_input; @@ -47,6 +48,12 @@ struct PreparedRow { golden: String, } +#[derive(Clone, Copy, Debug)] +struct SampleResult { + is_correct: bool, + response_parsed: bool, +} + /// Arguments for the [`CustomNoCodeBuiltin::evaluate_row`] method struct EvaluateRowArgs<'a> { /// The row index @@ -83,7 +90,7 @@ impl CustomNoCodeBuiltin { Self { name } } - async fn evaluate_row(&self, args: EvaluateRowArgs<'_>) -> Result { + async fn evaluate_row(&self, args: EvaluateRowArgs<'_>) -> Result { let prepared = prepare_row(args.i, args.row, args.style)?; let rendered = args @@ -147,9 +154,25 @@ impl CustomNoCodeBuiltin { None, ) .await; + args.metrics_store + .emit( + args.run_id, + Some(step_id), + "response_parsed", + if output.parsed_response.is_some() { + 1.0 + } else { + 0.0 + }, + None, + ) + .await; } - Ok(output.is_correct) + Ok(SampleResult { + is_correct: output.is_correct, + response_parsed: output.parsed_response.is_some(), + }) } } @@ -222,7 +245,7 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { .await?; let total_count = results.len(); - emit_accuracy_metrics(ctx.metrics_store, ctx.run_id, results).await; + emit_custom_nocode_aggregate_metrics(ctx.metrics_store, ctx.run_id, &results).await?; set_builtin_run_output(ctx.db, ctx.run_id, total_count).await?; @@ -230,6 +253,85 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { } } +#[expect(clippy::cast_precision_loss)] +async fn emit_custom_nocode_aggregate_metrics( + metrics_store: &crate::metrics_store::MetricsStore, + run_id: i64, + results: &[SampleResult], +) -> Result<()> { + if results.is_empty() { + return Ok(()); + } + + emit_accuracy_metrics( + metrics_store, + run_id, + results.iter().map(|result| result.is_correct), + ) + .await; + + let incorrect_count = results.iter().filter(|result| !result.is_correct).count(); + let parsed_response_count = results + .iter() + .filter(|result| result.response_parsed) + .count(); + let unparsed_response_count = results.len() - parsed_response_count; + let parse_rate = parsed_response_count as f64 / results.len() as f64; + + metrics_store + .emit( + run_id, + None, + "incorrect_count", + incorrect_count as f64, + None, + ) + .await; + metrics_store + .emit( + run_id, + None, + "parsed_response_count", + parsed_response_count as f64, + None, + ) + .await; + metrics_store + .emit( + run_id, + None, + "unparsed_response_count", + unparsed_response_count as f64, + None, + ) + .await; + metrics_store + .emit(run_id, None, "parse_rate", parse_rate, None) + .await; + + let latency_values = metrics_store + .sample_metric_values(run_id, "latency_ms") + .await?; + if latency_values.is_empty() { + return Ok(()); + } + let stats = compute_statistics(&latency_values); + for (metric_name, metric_value) in [ + ("mean_latency_ms", stats.mean), + ("median_latency_ms", stats.median), + ("p95_latency_ms", stats.p95), + ("p99_latency_ms", stats.p99), + ("min_latency_ms", stats.min), + ("max_latency_ms", stats.max), + ] { + metrics_store + .emit(run_id, None, metric_name, metric_value, Some("ms")) + .await; + } + + Ok(()) +} + fn resolve_custom_nocode_sampler( model: Option<&crate::llm::Sampler>, style: &crate::config::CustomNoCodeStyleConfig, @@ -625,6 +727,67 @@ mod tests { } } + #[tokio::test] + async fn emits_extended_aggregate_metrics() { + let tmpdir = tempfile::tempdir().unwrap(); + let metrics_store = + crate::metrics_store::MetricsStore::new(tmpdir.path().join("metrics")).unwrap(); + let run_id = 17; + for (step_id, latency) in [(1, 10.0), (2, 20.0), (3, 30.0)] { + metrics_store + .emit(run_id, Some(step_id), "latency_ms", latency, Some("ms")) + .await; + } + let results = [ + SampleResult { + is_correct: true, + response_parsed: true, + }, + SampleResult { + is_correct: false, + response_parsed: false, + }, + SampleResult { + is_correct: false, + response_parsed: true, + }, + ]; + + emit_custom_nocode_aggregate_metrics(&metrics_store, run_id, &results) + .await + .unwrap(); + metrics_store.flush(run_id).await.unwrap(); + let metrics = metrics_store.list_aggregate_for_run(run_id).await.unwrap(); + let value = |name: &str| { + metrics + .iter() + .find(|metric| metric.metric_name == name) + .unwrap() + .metric_value + }; + let assert_metric = |name: &str, expected: f64| { + let actual = value(name); + assert!( + (actual - expected).abs() < 1e-12, + "expected {name}={expected}, got {actual}" + ); + }; + + assert_metric("accuracy", 1.0 / 3.0); + assert_metric("correct_count", 1.0); + assert_metric("incorrect_count", 2.0); + assert_metric("total_count", 3.0); + assert_metric("parsed_response_count", 2.0); + assert_metric("unparsed_response_count", 1.0); + assert_metric("parse_rate", 2.0 / 3.0); + assert_metric("mean_latency_ms", 20.0); + assert_metric("median_latency_ms", 20.0); + assert_metric("p95_latency_ms", 29.0); + assert_metric("p99_latency_ms", 29.8); + assert_metric("min_latency_ms", 10.0); + assert_metric("max_latency_ms", 30.0); + } + fn multiple_choice_config( choices: crate::config::CustomNoCodeChoiceSource, answer: crate::config::CustomNoCodeAnswerSource, @@ -885,7 +1048,17 @@ mod tests { let names: Vec<&str> = agg.iter().map(|m| m.metric_name.as_str()).collect(); assert!(names.contains(&"accuracy")); assert!(names.contains(&"correct_count")); + assert!(names.contains(&"incorrect_count")); assert!(names.contains(&"total_count")); + assert!(names.contains(&"parsed_response_count")); + assert!(names.contains(&"unparsed_response_count")); + assert!(names.contains(&"parse_rate")); + assert!(names.contains(&"mean_latency_ms")); + assert!(names.contains(&"median_latency_ms")); + assert!(names.contains(&"p95_latency_ms")); + assert!(names.contains(&"p99_latency_ms")); + assert!(names.contains(&"min_latency_ms")); + assert!(names.contains(&"max_latency_ms")); let total_metric = agg.iter().find(|m| m.metric_name == "total_count").unwrap(); assert_eq!(total_metric.metric_value as i64, 2); @@ -904,6 +1077,11 @@ mod tests { .filter(|m| m.metric_name == "is_correct") .count(); assert_eq!(is_correct_count, 2); + let response_parsed_count = all_metrics + .iter() + .filter(|m| m.metric_name == "response_parsed") + .count(); + assert_eq!(response_parsed_count, 2); // Verify steps were persisted in SQLite. let steps = crate::db::list_steps_for_run(&db, run_id).await.unwrap(); diff --git a/cli/src/metrics_store.rs b/cli/src/metrics_store.rs index 2a5a9f1..6086679 100644 --- a/cli/src/metrics_store.rs +++ b/cli/src/metrics_store.rs @@ -184,6 +184,33 @@ impl MetricsStore { self.query_metrics(run_id, true).await } + /// Return sample-level values for one metric from persisted and currently buffered points. + /// Aggregate points with the same name are excluded. + pub(crate) async fn sample_metric_values( + &self, + run_id: i64, + metric_name: &str, + ) -> Result> { + let mut values: Vec = self + .list_for_run(run_id) + .await? + .into_iter() + .filter(|metric| metric.step_id.is_some() && metric.metric_name.as_str() == metric_name) + .map(|metric| metric.metric_value) + .collect(); + + let buffer = self.buffer.lock().await; + values.extend( + buffer + .get(&run_id) + .into_iter() + .flatten() + .filter(|metric| metric.step_id.is_some() && metric.metric_name == metric_name) + .map(|metric| metric.metric_value), + ); + Ok(values) + } + async fn query_metrics( &self, run_id: i64, @@ -326,6 +353,32 @@ mod tests { Ok(()) } + #[tokio::test] + async fn sample_metric_values_combines_persisted_and_buffered_points() -> Result<()> { + let tmpdir = tempfile::tempdir()?; + let root = tmpdir.path(); + init_workspace(root).await?; + let db = open_workspace(root).await?; + let store = MetricsStore::new(crate::db::metrics_dir(root))?; + + let run_id = create_run(&db, "demo", None).await?; + store + .emit(run_id, Some(1), "latency_ms", 10.0, Some("ms")) + .await; + store + .emit(run_id, None, "latency_ms", 999.0, Some("ms")) + .await; + store.flush(run_id).await?; + store + .emit(run_id, Some(2), "latency_ms", 20.0, Some("ms")) + .await; + + let values = store.sample_metric_values(run_id, "latency_ms").await?; + assert_eq!(values, vec![10.0, 20.0]); + + Ok(()) + } + #[tokio::test] async fn flush_no_emit_is_noop() -> Result<()> { let tmpdir = tempfile::tempdir()?;