From 5f080394fbc44cfbcc8eb0117913c1e9c1abd687 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Thu, 2 Jul 2026 09:52:44 -0700 Subject: [PATCH 01/20] Implementing custom-nocode benchmarks for QA-style benchmarks --- cli/Cargo.lock | 70 +++++++ cli/Cargo.toml | 1 + cli/src/builtins/custom_nocode.rs | 282 ++++++++++++++++++++++++++ cli/src/builtins/mod.rs | 5 +- cli/src/builtins/pubmedqa/mod.rs | 4 +- cli/src/builtins/similarity.rs | 4 +- cli/src/commands/run.rs | 76 ++++++- cli/src/config.rs | 158 ++++++++++++++- custom-nocode-examples/prompts/qa.txt | 1 + custom-nocode-examples/quantiles.toml | 9 + 10 files changed, 600 insertions(+), 10 deletions(-) create mode 100644 cli/src/builtins/custom_nocode.rs create mode 100644 custom-nocode-examples/prompts/qa.txt create mode 100644 custom-nocode-examples/quantiles.toml diff --git a/cli/Cargo.lock b/cli/Cargo.lock index c34372b..3c83a5f 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -3074,6 +3074,19 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jinja" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4eef51ad9c68d377a1604706ae4121b1b766c1504c86cb5247f4b6946cba9f1" +dependencies = [ + "indexmap 2.14.0", + "minijinja", + "minijinja-contrib", + "serde", + "serde_json", +] + [[package]] name = "jni" version = "0.22.4" @@ -3411,6 +3424,12 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" +[[package]] +name = "memo-map" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d1115007560874e373613744c6fba374c17688327a71c1476d1a5954cc857b" + [[package]] name = "memoffset" version = "0.9.1" @@ -3436,6 +3455,35 @@ dependencies = [ "unicase", ] +[[package]] +name = "minijinja" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3d648e68cea56d9858d535ee28f9538404e2dd8cb08ed0bd05dca379477f39" +dependencies = [ + "aho-corasick", + "indexmap 2.14.0", + "memo-map", + "percent-encoding", + "serde", + "serde_json", + "unicase", + "unicode-ident", + "v_htmlescape", +] + +[[package]] +name = "minijinja-contrib" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85342f6fac0be8ccd5bd00d9066be538f34f393f577b75d81b17c8398a6b43bb" +dependencies = [ + "minijinja", + "serde", + "textwrap", + "unicode_categories", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -4296,6 +4344,7 @@ dependencies = [ "fastembed", "futures", "genai", + "jinja", "parquet", "predicates", "rand 0.8.6", @@ -5399,6 +5448,12 @@ dependencies = [ "serde", ] +[[package]] +name = "smawk" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8e2fb0f499abb4d162f2bedad68f5ef91a1682b5a03596ddb67efd37768d100" + [[package]] name = "snafu" version = "0.8.9" @@ -5871,6 +5926,15 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "smawk", +] + [[package]] name = "thiserror" version = "2.0.18" @@ -6444,6 +6508,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "v_htmlescape" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e8257fbc510f0a46eb602c10215901938b5c2a7d5e70fc11483b1d3c9b5b18c" + [[package]] name = "value-ext" version = "0.1.3" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index efd9c27..631f074 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -15,6 +15,7 @@ dirs = "6" fastembed = "5" futures = "0.3" genai = "0.6.5" +jinja = "0.1" parquet = { version = "54", features = ["arrow"] } rand = "0.8" reqwest = { version = "0.13.3", features = ["json", "rustls"] } diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs new file mode 100644 index 0000000..10503f5 --- /dev/null +++ b/cli/src/builtins/custom_nocode.rs @@ -0,0 +1,282 @@ +use std::sync::Arc; + +use anyhow::{Context, Result, bail}; +use serde::{Deserialize, Serialize}; + +use crate::builtins::common::{get_max_workers, extract_text, hash_input, run_timed_step}; +use crate::builtins::dataset_runner::DatasetRunner; +use crate::builtins::input::set_builtin_run_input; +use crate::builtins::output::set_builtin_run_output; +use crate::builtins::{BuiltinContext, BuiltinWorkflow}; +use crate::dataset::DatasetManager; +use crate::llm::LLMSampler; +use crate::llm::random::RandomSampler; + +/// Input deserialized from the JSON assembled by `commands::run`. +#[derive(Debug, Deserialize)] +struct CustomNoCodeInput { + style: String, + dataset: String, + #[serde(default)] + model: Option, + prompt_template_file: String, + prompt_column: String, + golden_column: String, + #[serde(default)] + limit: Option, + #[serde(default)] + max_workers: Option, +} + +/// Per-row step output stored as JSON in the step record. +#[derive(Debug, Serialize, Deserialize)] +struct RowOutput { + input: String, + response: String, + golden: String, + is_correct: bool, +} + +/// No-code custom benchmark builtin. +pub struct CustomNoCodeBuiltin { + name: String, +} + +impl CustomNoCodeBuiltin { + /// Create a new builtin with the workflow name from the config file. + pub fn new(name: String) -> Self { + Self { name } + } +} + +#[async_trait::async_trait] +impl BuiltinWorkflow for CustomNoCodeBuiltin { + fn name(&self) -> String { + self.name.clone() + } + + #[expect(clippy::too_many_lines)] + async fn execute(&self, ctx: BuiltinContext<'_>) -> Result<()> { + let config: CustomNoCodeInput = ctx + .input + .map(serde_json::from_str) + .transpose() + .context("invalid builtin input JSON")? + .unwrap_or_default(); + + if config.style != "qa" { + bail!( + "unsupported custom_nocode style `{}`; only `qa` is supported", + config.style + ); + } + + if config.limit == Some(0) { + bail!("limit must be > 0"); + } + + // Load template file and validate syntax. + let template_str = std::fs::read_to_string(&config.prompt_template_file) + .with_context(|| { + format!( + "failed to read prompt template file `{}`", + config.prompt_template_file + ) + })?; + let env = jinja::Environment::new(); + env.render_str(&template_str, jinja::context!(prompt => "")) + .with_context(|| { + format!( + "invalid jinja syntax in prompt template file `{}`", + config.prompt_template_file + ) + })?; + + let max_workers = config.max_workers.unwrap_or_else(get_max_workers); + + let llm: Arc = match config.model { + None => Arc::new(RandomSampler::new(80)), + Some(ref sampler) => sampler.resolve()?, + }; + + let manager = DatasetManager::new()?; + let info = manager + .init(&config.dataset, None, None, None) + .await?; + + let total = info + .total_rows + .context("could not determine dataset size; pass an explicit limit")?; + let limit = config.limit.unwrap_or(total).min(total); + + set_builtin_run_input( + ctx.db, + ctx.run_id, + config.model.as_ref(), + limit, + config.max_workers, + ) + .await?; + + let db = ctx.db.clone(); + let model = config.model.clone(); + let run_id = ctx.run_id; + let dataset = config.dataset.clone(); + let prompt_column = config.prompt_column.clone(); + let golden_column = config.golden_column.clone(); + let template_str = Arc::new(template_str); + + let results = DatasetRunner::new(&manager, &dataset, &info, limit) + .desc(&self.name()) + .set_quiet(ctx.quiet) + .for_each_concurrent(max_workers, move |i, row| { + let llm = Arc::clone(&llm); + let db = db.clone(); + let model = model.clone(); + let template_str = Arc::clone(&template_str); + let prompt_column = prompt_column.clone(); + let golden_column = golden_column.clone(); + async move { + let prompt = extract_text(&row, &prompt_column) + .with_context(|| { + format!("row {i}: missing prompt column `{prompt_column}`") + })?; + let golden = extract_text(&row, &golden_column) + .with_context(|| { + format!("row {i}: missing golden column `{golden_column}`") + })?; + + let rendered = env + .render_str(&template_str, jinja::context!(prompt => &prompt)) + .with_context(|| { + format!("row {i}: failed to render prompt template") + })?; + + let model_str = model + .as_ref() + .map_or("random".to_string(), std::string::ToString::to_string); + let input_hash = hash_input(&format!( + "{rendered}\nmodel={model_str}\nworkflow={}" + , self.name())); + let step_key = format!("row-{i}"); + + let (output, step_id) = run_timed_step( + &db, + ctx.metrics_store, + run_id, + &step_key, + &input_hash, + async { + let model_response = llm + .sample(&rendered) + .await + .with_context(|| { + format!("failed to sample LLM for row {i}") + })?; + + let is_correct = + model_response.trim() == golden.trim(); + + Ok(RowOutput { + input: rendered.clone(), + response: model_response, + golden, + is_correct, + }) + }, + ) + .await?; + + if let Some(step_id) = step_id { + ctx.metrics_store + .emit( + ctx.run_id, + Some(step_id), + "is_correct", + if output.is_correct { 1.0 } else { 0.0 }, + None, + ) + .await; + } + + Ok::<_, anyhow::Error>(output.is_correct) + } + }) + .await?; + + let mut correct_count: usize = 0; + let total_count = results.len(); + for is_correct in results { + if is_correct { + correct_count += 1; + } + } + + #[expect(clippy::cast_precision_loss)] + if total_count > 0 { + let accuracy = correct_count as f64 / total_count as f64; + + ctx.metrics_store + .emit(ctx.run_id, None, "accuracy", accuracy, None) + .await; + ctx.metrics_store + .emit( + ctx.run_id, + None, + "correct_count", + correct_count as f64, + None, + ) + .await; + ctx.metrics_store + .emit(ctx.run_id, None, "total_count", total_count as f64, None) + .await; + } + + set_builtin_run_output(ctx.db, ctx.run_id, total_count).await?; + + Ok(()) + } +} + +/// Case-sensitive exact-match comparison after trimming whitespace. +fn is_exact_match(response: &str, golden: &str) -> bool { + response.trim() == golden.trim() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn render_template_with_prompt_variable() { + let template = "Answer this: {{ prompt }}"; + let env = jinja::Environment::new(); + let rendered = env + .render_str(template, jinja::context!(prompt => "what is 2+2")) + .unwrap(); + assert_eq!(rendered, "Answer this: what is 2+2"); + } + + #[test] + fn render_template_preserves_newlines() { + let template = "Question:\n{{ prompt }}\nAnswer:"; + let env = jinja::Environment::new(); + let rendered = env + .render_str(template, jinja::context!(prompt => "hello")) + .unwrap(); + assert_eq!(rendered, "Question:\nhello\nAnswer:"); + } + + #[test] + fn exact_match_case_sensitive() { + assert!(is_exact_match("hello", "hello")); + assert!(!is_exact_match("Hello", "hello")); + } + + #[test] + fn exact_match_trims_whitespace() { + assert!(is_exact_match(" hello ", "hello")); + assert!(is_exact_match("hello", " hello ")); + } +} diff --git a/cli/src/builtins/mod.rs b/cli/src/builtins/mod.rs index 0611a72..3a364b4 100644 --- a/cli/src/builtins/mod.rs +++ b/cli/src/builtins/mod.rs @@ -1,4 +1,5 @@ mod common; +mod custom_nocode; mod dataset_runner; mod financebench; mod input; @@ -7,6 +8,8 @@ mod pubmedqa; mod similarity; mod simpleqa_verified; +pub use custom_nocode::CustomNoCodeBuiltin; + use anyhow::Result; use async_trait::async_trait; use sea_orm::DatabaseConnection; @@ -28,7 +31,7 @@ pub struct BuiltinContext<'a> { #[async_trait] pub trait BuiltinWorkflow: Send + Sync { /// Unique name of the builtin (e.g. "financebench"). - fn name(&self) -> &'static str; + fn name(&self) -> String; /// Execute the builtin eval. async fn execute(&self, ctx: BuiltinContext<'_>) -> Result<()>; } diff --git a/cli/src/builtins/pubmedqa/mod.rs b/cli/src/builtins/pubmedqa/mod.rs index 7ea162c..a7802e7 100644 --- a/cli/src/builtins/pubmedqa/mod.rs +++ b/cli/src/builtins/pubmedqa/mod.rs @@ -25,8 +25,8 @@ pub struct PubmedqaBuiltin; #[expect(clippy::too_many_lines)] #[async_trait::async_trait] impl BuiltinWorkflow for PubmedqaBuiltin { - fn name(&self) -> &'static str { - "pubmedqa" + fn name(&self) -> String { + "pubmedqa".to_string() } async fn execute(&self, ctx: BuiltinContext<'_>) -> Result<()> { diff --git a/cli/src/builtins/similarity.rs b/cli/src/builtins/similarity.rs index 051a260..f6d151a 100644 --- a/cli/src/builtins/similarity.rs +++ b/cli/src/builtins/similarity.rs @@ -69,8 +69,8 @@ pub const FINANCEBENCH: SimilarityBenchmark = SimilarityBenchmark { #[expect(clippy::too_many_lines)] #[async_trait::async_trait] impl BuiltinWorkflow for SimilarityBenchmark { - fn name(&self) -> &'static str { - self.name + fn name(&self) -> String { + self.name.to_string() } async fn execute(&self, ctx: BuiltinContext<'_>) -> Result<()> { diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs index 18fcb06..7b31117 100644 --- a/cli/src/commands/run.rs +++ b/cli/src/commands/run.rs @@ -75,6 +75,35 @@ pub async fn run( ) .await } + qt::config::BenchmarkConfig::CustomNoCode(c) => { + let input = assemble_custom_nocode_input(c, cli_input); + + let cwd = std::env::current_dir()?; + let root = db::resolve_workspace_root(&cwd, true).await?; + let db = db::open_workspace(&root).await?; + let metrics_store = MetricsStore::new(db::metrics_dir(&root))?; + let run_id = db::create_run(&db, workflow_name, input.as_deref()).await?; + + if !json { + println!("Created run {run_id}"); + } + + let builtin = + Box::new(qt::builtins::CustomNoCodeBuiltin::new( + workflow_name.to_owned(), + )); + execute_builtin( + &db, + &metrics_store, + run_id, + workflow_name, + builtin, + input.as_deref(), + json, + process_start, + ) + .await + } } } None => { @@ -122,6 +151,46 @@ fn assemble_builtin_input( } } +/// Config input shape for `custom_nocode` benchmarks auto-generated from +/// `quantiles.toml` `[benchmarks.*]`. +#[derive(Serialize)] +struct CustomNoCodeConfigInput<'a> { + style: &'a str, + dataset: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + model: &'a Option, + prompt_template_file: &'a str, + prompt_column: &'a str, + golden_column: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + max_workers: Option, +} + +fn assemble_custom_nocode_input( + bench: &qt::config::CustomNoCodeBenchmarkConfig, + cli_input: Option<&str>, +) -> Option { + if let Some(cli_str) = cli_input { + return Some(cli_str.to_owned()); + } + + let input = CustomNoCodeConfigInput { + style: "qa", + dataset: &bench.dataset, + model: &bench.model, + prompt_template_file: &bench.qa.prompt_template_file, + prompt_column: &bench.qa.prompt_column, + golden_column: &bench.qa.golden_column, + limit: bench.qa.limit, + max_workers: bench.qa.max_workers, + }; + + let json = serde_json::to_string(&input).expect("infallible serialization"); + Some(json) +} + fn merge_inputs( config_input: Option<&HashMap>, cli_input: Option<&str>, @@ -162,6 +231,9 @@ async fn run_builtin_workflow( json: bool, process_start: Instant, ) -> Result<()> { + let builtin = builtins::resolve(workflow_name) + .with_context(|| format!("builtin `{workflow_name}` not found"))?; + let cwd = std::env::current_dir()?; let root = db::resolve_workspace_root(&cwd, true).await?; let db = db::open_workspace(&root).await?; @@ -177,6 +249,7 @@ async fn run_builtin_workflow( &metrics_store, run_id, workflow_name, + builtin, input, json, process_start, @@ -189,12 +262,11 @@ pub async fn execute_builtin( metrics_store: &MetricsStore, run_id: i64, workflow_name: &str, + builtin: Box, input: Option<&str>, json: bool, process_start: Instant, ) -> Result<()> { - let builtin = builtins::resolve(workflow_name) - .with_context(|| format!("builtin `{workflow_name}` not found"))?; let builtin_result = builtin .execute(builtins::BuiltinContext { diff --git a/cli/src/config.rs b/cli/src/config.rs index 0da4a18..0ecc773 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -7,12 +7,13 @@ use crate::llm::Sampler; /// Configuration for a single benchmark. /// -/// Exactly one of the two variants is deserialized based on the `type` field: -/// `builtin` (default when absent) or `custom_code`. +/// Exactly one of the variants is deserialized based on the `type` field: +/// `builtin` (default when absent), `custom_code`, or `custom_nocode`. #[derive(Debug, Clone)] pub enum BenchmarkConfig { Builtin(BuiltinBenchmarkConfig), CustomCode(CustomCodeBenchmarkConfig), + CustomNoCode(CustomNoCodeBenchmarkConfig), } impl BenchmarkConfig { @@ -30,6 +31,15 @@ impl BenchmarkConfig { } Ok(()) } + BenchmarkConfig::CustomNoCode(c) => { + if !std::path::Path::new(&c.qa.prompt_template_file).is_file() { + bail!( + "custom_nocode benchmark config `prompt_template_file` must point to an existing file; `{}` not found", + c.qa.prompt_template_file + ); + } + Ok(()) + } } } } @@ -53,6 +63,14 @@ impl<'de> Deserialize<'de> for BenchmarkConfig { })?; Ok(BenchmarkConfig::CustomCode(config)) } + Some("custom_nocode") => { + let config = CustomNoCodeBenchmarkConfig::deserialize(value).map_err(|e| { + serde::de::Error::custom(format!( + "failed to deserialize custom_nocode benchmark config: {e}" + )) + })?; + Ok(BenchmarkConfig::CustomNoCode(config)) + } Some("builtin") | None => { let config = BuiltinBenchmarkConfig::deserialize(value).map_err(|e| { serde::de::Error::custom(format!( @@ -62,7 +80,7 @@ impl<'de> Deserialize<'de> for BenchmarkConfig { Ok(BenchmarkConfig::Builtin(config)) } Some(other) => Err(serde::de::Error::custom(format!( - "invalid benchmark type `{other}`; expected `builtin` or `custom_code`", + "invalid benchmark type `{other}`; expected `builtin`, `custom_code`, or `custom_nocode`", ))), } } @@ -98,6 +116,63 @@ pub struct CustomCodeBenchmarkConfig { pub input: Option>, } +/// Style of a no-code custom benchmark. +/// +/// Future variants may include: +/// - `Judge` – evaluate responses against a rubric using a judge model. +/// - `Similarity` – score responses with a similarity metric. +/// - `Agent` – run multi-step agent loops. +#[derive(Debug, Clone)] +pub enum CustomNoCodeStyle { + Qa, +} + +impl<'de> Deserialize<'de> for CustomNoCodeStyle { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + match s.as_str() { + "qa" => Ok(CustomNoCodeStyle::Qa), + other => Err(serde::de::Error::custom(format!( + "unsupported custom_nocode style `{other}`; expected `qa`", + ))), + } + } +} + +/// QA-specific configuration for a no-code custom benchmark. +#[derive(Debug, Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct CustomNoCodeQaConfig { + /// Path to a Jinja template file for rendering prompts. + pub prompt_template_file: String, + /// Dataset column containing the prompt text. + pub prompt_column: String, + /// Dataset column containing the golden answer. + pub golden_column: String, + /// Number of dataset rows to evaluate. + pub limit: Option, + /// Maximum concurrent workers. + pub max_workers: Option, +} + +/// No-code custom benchmark configuration. +#[derive(Debug, Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct CustomNoCodeBenchmarkConfig { + #[serde(rename = "type")] + pub type_: String, + pub style: CustomNoCodeStyle, + /// Dataset identifier (e.g. `quantiles/simpleqa-verified`). + pub dataset: String, + /// Model sampler to use. + pub model: Option, + #[serde(flatten)] + pub qa: CustomNoCodeQaConfig, +} + /// Top-level workspace configuration read from `quantiles.toml` or /// `.quantiles.toml` in the current working directory. #[derive(Debug, Deserialize, Default)] @@ -269,6 +344,25 @@ mod tests { assert!(err.to_string().contains("non-empty `command`")); } + #[test] + fn validate_rejects_missing_template_file() { + let bench = BenchmarkConfig::CustomNoCode(CustomNoCodeBenchmarkConfig { + type_: "custom_nocode".to_owned(), + style: CustomNoCodeStyle::Qa, + dataset: "quantiles/simpleqa-verified".to_owned(), + model: Some(Sampler::Random), + qa: CustomNoCodeQaConfig { + prompt_template_file: "does_not_exist.txt".to_owned(), + prompt_column: "problem".to_owned(), + golden_column: "answer".to_owned(), + limit: None, + max_workers: None, + }, + }); + let err = bench.validate().unwrap_err(); + assert!(err.to_string().contains("not found")); + } + #[test] fn validate_accepts_nonempty_command() { let bench = BenchmarkConfig::CustomCode(CustomCodeBenchmarkConfig { @@ -289,6 +383,64 @@ mod tests { assert!(result.is_err()); } + #[test] + fn deserialize_custom_nocode_qa() { + let toml = r#" + [benchmarks.nocode_custom] + type = "custom_nocode" + style = "qa" + dataset = "quantiles/simpleqa-verified" + model = "random" + prompt_template_file = "prompts/qa.txt" + prompt_column = "problem" + golden_column = "answer" + limit = 10 + "#; + let config: WorkspaceConfig = toml::from_str(toml).unwrap(); + let bench = config.benchmarks.get("nocode_custom").unwrap(); + assert!(matches!(bench, BenchmarkConfig::CustomNoCode(_))); + if let BenchmarkConfig::CustomNoCode(c) = bench { + assert!(matches!(c.style, CustomNoCodeStyle::Qa)); + assert_eq!(c.dataset, "quantiles/simpleqa-verified"); + assert_eq!(c.model, Some(Sampler::Random)); + assert_eq!(c.qa.prompt_template_file, "prompts/qa.txt"); + assert_eq!(c.qa.prompt_column, "problem"); + assert_eq!(c.qa.golden_column, "answer"); + assert_eq!(c.qa.limit, Some(10)); + } + } + + #[test] + fn deserialize_custom_nocode_unsupported_style_errors() { + let toml = r#" + [benchmarks.nocode_custom] + type = "custom_nocode" + style = "judge" + dataset = "quantiles/simpleqa-verified" + model = "random" + prompt_template_file = "prompts/qa.txt" + prompt_column = "problem" + golden_column = "answer" + "#; + let result: Result = toml::from_str(toml); + assert!(result.is_err()); + } + + #[test] + fn custom_nocode_missing_required_field_errors() { + let toml = r#" + [benchmarks.nocode_custom] + type = "custom_nocode" + style = "qa" + dataset = "quantiles/simpleqa-verified" + model = "random" + prompt_column = "problem" + golden_column = "answer" + "#; + let result: Result = toml::from_str(toml); + assert!(result.is_err()); + } + /// `custom_code` benchmarks must have a `command` field; omitting it should fail at /// parse time because the struct requires it. #[test] diff --git a/custom-nocode-examples/prompts/qa.txt b/custom-nocode-examples/prompts/qa.txt new file mode 100644 index 0000000..c4f9438 --- /dev/null +++ b/custom-nocode-examples/prompts/qa.txt @@ -0,0 +1 @@ +{{ prompt }} diff --git a/custom-nocode-examples/quantiles.toml b/custom-nocode-examples/quantiles.toml new file mode 100644 index 0000000..1372044 --- /dev/null +++ b/custom-nocode-examples/quantiles.toml @@ -0,0 +1,9 @@ +[benchmarks.nocode_custom] +type = "custom_nocode" +style = "qa" +dataset = "quantiles/simpleqa-verified" +model = "random" +prompt_template_file = "prompts/qa.txt" +prompt_column = "problem" +golden_column = "answer" +limit = 10 From 9d25c8976123fe908293ff64ecfc0d8755c86cc3 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Thu, 2 Jul 2026 09:57:10 -0700 Subject: [PATCH 02/20] more progress --- cli/src/builtins/custom_nocode.rs | 9 +++++---- cli/src/builtins/pubmedqa/mod.rs | 3 ++- cli/src/commands/resume.rs | 13 ++++++++++++- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index 10503f5..b41182e 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -13,7 +13,7 @@ use crate::llm::LLMSampler; use crate::llm::random::RandomSampler; /// Input deserialized from the JSON assembled by `commands::run`. -#[derive(Debug, Deserialize)] +#[derive(Debug, Default, Deserialize)] struct CustomNoCodeInput { style: String, dataset: String, @@ -126,8 +126,9 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { let golden_column = config.golden_column.clone(); let template_str = Arc::new(template_str); + let name = self.name(); let results = DatasetRunner::new(&manager, &dataset, &info, limit) - .desc(&self.name()) + .desc(&name) .set_quiet(ctx.quiet) .for_each_concurrent(max_workers, move |i, row| { let llm = Arc::clone(&llm); @@ -136,6 +137,7 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { let template_str = Arc::clone(&template_str); let prompt_column = prompt_column.clone(); let golden_column = golden_column.clone(); + let env = env.clone(); async move { let prompt = extract_text(&row, &prompt_column) .with_context(|| { @@ -174,8 +176,7 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { format!("failed to sample LLM for row {i}") })?; - let is_correct = - model_response.trim() == golden.trim(); + let is_correct = is_exact_match(&model_response, &golden); Ok(RowOutput { input: rendered.clone(), diff --git a/cli/src/builtins/pubmedqa/mod.rs b/cli/src/builtins/pubmedqa/mod.rs index a7802e7..60242ad 100644 --- a/cli/src/builtins/pubmedqa/mod.rs +++ b/cli/src/builtins/pubmedqa/mod.rs @@ -72,8 +72,9 @@ impl BuiltinWorkflow for PubmedqaBuiltin { let model = config.base.model.clone(); let run_id = ctx.run_id; + let name = self.name(); let results = DatasetRunner::new(&manager, dataset_id, &info, limit) - .desc(self.name()) + .desc(&name) .set_quiet(ctx.quiet) .for_each_concurrent(max_workers, move |i, row| { let llm = Arc::clone(&llm); diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs index 91b61af..e4123ad 100644 --- a/cli/src/commands/resume.rs +++ b/cli/src/commands/resume.rs @@ -1,6 +1,6 @@ use std::time::Instant; -use anyhow::{Result, bail}; +use anyhow::{Context, Result, bail}; use qt::builtins; use qt::db; @@ -40,6 +40,7 @@ pub(crate) fn plan_resume( qt::config::BenchmarkConfig::CustomCode(c) => { Ok(ResumePlan::CustomCode(c.command.clone())) } + qt::config::BenchmarkConfig::CustomNoCode(_) => Ok(ResumePlan::Builtin), } } None => { @@ -95,11 +96,21 @@ pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<( // wise to revisit this policy. match plan { ResumePlan::Builtin => { + let builtin: Box = match bench_config { + Some(qt::config::BenchmarkConfig::CustomNoCode(_)) => { + Box::new(builtins::CustomNoCodeBuiltin::new( + workflow_name.to_owned(), + )) + } + _ => builtins::resolve(workflow_name) + .with_context(|| format!("builtin `{workflow_name}` not found"))?, + }; super::run::execute_builtin( &db, &metrics_store, run_id, workflow_name, + builtin, stored_input, json, process_start, From 5214d97d31c4dc08600789e3fe3eab567f302975 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Thu, 2 Jul 2026 09:58:29 -0700 Subject: [PATCH 03/20] fmt --- cli/src/builtins/custom_nocode.rs | 38 +++++++++++++------------------ cli/src/commands/resume.rs | 4 +--- cli/src/commands/run.rs | 8 +++---- 3 files changed, 20 insertions(+), 30 deletions(-) diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index b41182e..b4638b6 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; -use crate::builtins::common::{get_max_workers, extract_text, hash_input, run_timed_step}; +use crate::builtins::common::{extract_text, get_max_workers, hash_input, run_timed_step}; use crate::builtins::dataset_runner::DatasetRunner; use crate::builtins::input::set_builtin_run_input; use crate::builtins::output::set_builtin_run_output; @@ -44,6 +44,7 @@ pub struct CustomNoCodeBuiltin { impl CustomNoCodeBuiltin { /// Create a new builtin with the workflow name from the config file. + #[must_use] pub fn new(name: String) -> Self { Self { name } } @@ -76,8 +77,8 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { } // Load template file and validate syntax. - let template_str = std::fs::read_to_string(&config.prompt_template_file) - .with_context(|| { + let template_str = + std::fs::read_to_string(&config.prompt_template_file).with_context(|| { format!( "failed to read prompt template file `{}`", config.prompt_template_file @@ -100,9 +101,7 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { }; let manager = DatasetManager::new()?; - let info = manager - .init(&config.dataset, None, None, None) - .await?; + let info = manager.init(&config.dataset, None, None, None).await?; let total = info .total_rows @@ -139,27 +138,24 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { let golden_column = golden_column.clone(); let env = env.clone(); async move { - let prompt = extract_text(&row, &prompt_column) - .with_context(|| { - format!("row {i}: missing prompt column `{prompt_column}`") - })?; - let golden = extract_text(&row, &golden_column) - .with_context(|| { - format!("row {i}: missing golden column `{golden_column}`") - })?; + let prompt = extract_text(&row, &prompt_column).with_context(|| { + format!("row {i}: missing prompt column `{prompt_column}`") + })?; + let golden = extract_text(&row, &golden_column).with_context(|| { + format!("row {i}: missing golden column `{golden_column}`") + })?; let rendered = env .render_str(&template_str, jinja::context!(prompt => &prompt)) - .with_context(|| { - format!("row {i}: failed to render prompt template") - })?; + .with_context(|| format!("row {i}: failed to render prompt template"))?; let model_str = model .as_ref() .map_or("random".to_string(), std::string::ToString::to_string); let input_hash = hash_input(&format!( - "{rendered}\nmodel={model_str}\nworkflow={}" - , self.name())); + "{rendered}\nmodel={model_str}\nworkflow={}", + self.name() + )); let step_key = format!("row-{i}"); let (output, step_id) = run_timed_step( @@ -172,9 +168,7 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { let model_response = llm .sample(&rendered) .await - .with_context(|| { - format!("failed to sample LLM for row {i}") - })?; + .with_context(|| format!("failed to sample LLM for row {i}"))?; let is_correct = is_exact_match(&model_response, &golden); diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs index e4123ad..fcd797b 100644 --- a/cli/src/commands/resume.rs +++ b/cli/src/commands/resume.rs @@ -98,9 +98,7 @@ pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<( ResumePlan::Builtin => { let builtin: Box = match bench_config { Some(qt::config::BenchmarkConfig::CustomNoCode(_)) => { - Box::new(builtins::CustomNoCodeBuiltin::new( - workflow_name.to_owned(), - )) + Box::new(builtins::CustomNoCodeBuiltin::new(workflow_name.to_owned())) } _ => builtins::resolve(workflow_name) .with_context(|| format!("builtin `{workflow_name}` not found"))?, diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs index 7b31117..a373fa3 100644 --- a/cli/src/commands/run.rs +++ b/cli/src/commands/run.rs @@ -88,10 +88,9 @@ pub async fn run( println!("Created run {run_id}"); } - let builtin = - Box::new(qt::builtins::CustomNoCodeBuiltin::new( - workflow_name.to_owned(), - )); + let builtin = Box::new(qt::builtins::CustomNoCodeBuiltin::new( + workflow_name.to_owned(), + )); execute_builtin( &db, &metrics_store, @@ -267,7 +266,6 @@ pub async fn execute_builtin( json: bool, process_start: Instant, ) -> Result<()> { - let builtin_result = builtin .execute(builtins::BuiltinContext { db, From 525b2e6ef331664261266bf96f5ce9aa01b8931f Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:00:54 -0700 Subject: [PATCH 04/20] progress --- cli/src/commands/resume.rs | 4 ++-- cli/src/commands/run.rs | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs index fcd797b..b8e5585 100644 --- a/cli/src/commands/resume.rs +++ b/cli/src/commands/resume.rs @@ -36,11 +36,11 @@ pub(crate) fn plan_resume( Some(bench) => { bench.validate()?; match bench { - qt::config::BenchmarkConfig::Builtin(_) => Ok(ResumePlan::Builtin), qt::config::BenchmarkConfig::CustomCode(c) => { Ok(ResumePlan::CustomCode(c.command.clone())) } - qt::config::BenchmarkConfig::CustomNoCode(_) => Ok(ResumePlan::Builtin), + qt::config::BenchmarkConfig::Builtin(_) + | qt::config::BenchmarkConfig::CustomNoCode(_) => Ok(ResumePlan::Builtin), } } None => { diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs index a373fa3..f8ffb04 100644 --- a/cli/src/commands/run.rs +++ b/cli/src/commands/run.rs @@ -82,7 +82,7 @@ pub async fn run( let root = db::resolve_workspace_root(&cwd, true).await?; let db = db::open_workspace(&root).await?; let metrics_store = MetricsStore::new(db::metrics_dir(&root))?; - let run_id = db::create_run(&db, workflow_name, input.as_deref()).await?; + let run_id = db::create_run(&db, workflow_name, Some(&input)).await?; if !json { println!("Created run {run_id}"); @@ -97,7 +97,7 @@ pub async fn run( run_id, workflow_name, builtin, - input.as_deref(), + Some(&input), json, process_start, ) @@ -170,9 +170,9 @@ struct CustomNoCodeConfigInput<'a> { fn assemble_custom_nocode_input( bench: &qt::config::CustomNoCodeBenchmarkConfig, cli_input: Option<&str>, -) -> Option { +) -> String { if let Some(cli_str) = cli_input { - return Some(cli_str.to_owned()); + return cli_str.to_owned(); } let input = CustomNoCodeConfigInput { @@ -186,8 +186,7 @@ fn assemble_custom_nocode_input( max_workers: bench.qa.max_workers, }; - let json = serde_json::to_string(&input).expect("infallible serialization"); - Some(json) + serde_json::to_string(&input).expect("infallible serialization") } fn merge_inputs( @@ -256,6 +255,7 @@ async fn run_builtin_workflow( .await } +#[expect(clippy::too_many_arguments)] pub async fn execute_builtin( db: &DatabaseConnection, metrics_store: &MetricsStore, From da8aa77fdbe2048876680d8fb179d3a9dfa1ce02 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:47:04 -0700 Subject: [PATCH 05/20] test progress --- cli/src/builtins/custom_nocode.rs | 205 ++++++++++++++++++++++++++++++ cli/src/commands/resume.rs | 23 ++++ cli/src/commands/run.rs | 74 +++++++++++ cli/src/config.rs | 19 +++ cli/src/dataset/hf_client.rs | 21 ++- cli/src/dataset/mod.rs | 14 +- 6 files changed, 345 insertions(+), 11 deletions(-) diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index b4638b6..d95517d 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -242,6 +242,15 @@ fn is_exact_match(response: &str, golden: &str) -> bool { #[cfg(test)] mod tests { use super::*; + use serde_json::json; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + #[test] + fn builtin_name_returns_configured_name() { + let builtin = CustomNoCodeBuiltin::new("my-benchmark".to_owned()); + assert_eq!(builtin.name(), "my-benchmark"); + } #[test] fn render_template_with_prompt_variable() { @@ -274,4 +283,200 @@ mod tests { assert!(is_exact_match(" hello ", "hello")); assert!(is_exact_match("hello", " hello ")); } + + #[tokio::test] + async fn execute_rejects_invalid_jinja_template() { + let tmpdir = tempfile::tempdir().unwrap(); + let root = tmpdir.path(); + crate::db::init_workspace(root).await.unwrap(); + let db = crate::db::open_workspace(root).await.unwrap(); + let metrics_store = + crate::metrics_store::MetricsStore::new(crate::db::metrics_dir(root)).unwrap(); + + let template_path = root.join("bad.txt"); + std::fs::write(&template_path, "{{ unclosed").unwrap(); + + let input_json = serde_json::to_string(&json!({ + "style": "qa", + "dataset": "fixture/qa", + "prompt_template_file": template_path.to_str().unwrap(), + "prompt_column": "q", + "golden_column": "a", + })) + .unwrap(); + + let run_id = crate::db::create_run(&db, "test", Some(&input_json)) + .await + .unwrap(); + + let workflow_name = "test".to_owned(); + let builtin = CustomNoCodeBuiltin::new(workflow_name.clone()); + let result = builtin + .execute(super::BuiltinContext { + db: &db, + metrics_store: &metrics_store, + run_id, + workflow_name: &workflow_name, + input: Some(&input_json), + quiet: true, + }) + .await; + + assert!(result.is_err()); + let msg = result.unwrap_err().to_string(); + assert!( + msg.contains("invalid jinja syntax"), + "unexpected error: {msg}" + ); + } + + #[expect(clippy::too_many_lines)] + #[expect(clippy::cast_possible_truncation)] + #[tokio::test] + async fn execute_records_metrics_and_steps_with_fixture() { + let server = MockServer::start().await; + let tmpdir = tempfile::tempdir().unwrap(); + let root = tmpdir.path(); + let cache_dir = root.join("cache"); + + // Save original env var values so we can restore them later. + let orig_hf = std::env::var("HF_DATASETS_SERVER").ok(); + let orig_cache = std::env::var("QUANTILES_DATASET_CACHE_DIR").ok(); + unsafe { + std::env::set_var("HF_DATASETS_SERVER", server.uri()); + std::env::set_var("QUANTILES_DATASET_CACHE_DIR", cache_dir.as_os_str()); + } + + // Mock HF dataset server endpoints used by DatasetManager::init(). + Mock::given(method("GET")) + .and(path("/splits")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "splits": [{"config": "default", "split": "train"}] + }))) + .mount(&server) + .await; + + Mock::given(method("GET")) + .and(path("/size")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "size": {"splits": [{"num_rows": 2}]} + }))) + .mount(&server) + .await; + + // Initialize workspace with SQLite DB and metrics dir. + crate::db::init_workspace(root).await.unwrap(); + let db = crate::db::open_workspace(root).await.unwrap(); + let metrics_store = + crate::metrics_store::MetricsStore::new(crate::db::metrics_dir(root)).unwrap(); + + // Write a Jinja template file. + let template_path = root.join("template.txt"); + std::fs::write(&template_path, "{{ prompt }}\nAnswer:").unwrap(); + + // Pre-populate the dataset cache so no network fetch is needed for rows. + let cache = crate::dataset::cache::DatasetCache::new(cache_dir); + let rows = vec![ + json!({"question": "what is 2+2", "answer": "4"}), + json!({"question": "what is 3+3", "answer": "6"}), + ]; + let key = crate::dataset::cache::cache_key("fixture/qa", "default", "train", None); + let batch_path = cache.batch_path(&key, 0, 2); + cache.write_batch(&batch_path, &rows).await.unwrap(); + + // Assemble the input JSON that execute() expects. + let input_json = serde_json::to_string(&json!({ + "style": "qa", + "dataset": "fixture/qa", + "model": "random", + "prompt_template_file": template_path.to_str().unwrap(), + "prompt_column": "question", + "golden_column": "answer", + "limit": 2, + })) + .unwrap(); + + let run_id = crate::db::create_run(&db, "test_nocode", Some(&input_json)) + .await + .unwrap(); + + let workflow_name = "test_nocode".to_owned(); + let builtin = CustomNoCodeBuiltin::new(workflow_name.clone()); + builtin + .execute(super::BuiltinContext { + db: &db, + metrics_store: &metrics_store, + run_id, + workflow_name: &workflow_name, + input: Some(&input_json), + quiet: true, + }) + .await + .unwrap(); + + // Flush buffered metrics to Parquet so we can read them back. + metrics_store.flush(run_id).await.unwrap(); + + // Verify aggregate metrics were written. + let agg = metrics_store.list_aggregate_for_run(run_id).await.unwrap(); + let names: Vec<&str> = agg.iter().map(|m| m.metric_name.as_str()).collect(); + assert!(names.contains(&"accuracy")); + assert!(names.contains(&"correct_count")); + assert!(names.contains(&"total_count")); + + let total_metric = agg.iter().find(|m| m.metric_name == "total_count").unwrap(); + assert_eq!(total_metric.metric_value as i64, 2); + + // Random sampler responses won't match "4" or "6", so correctness is 0. + let correct_metric = agg + .iter() + .find(|m| m.metric_name == "correct_count") + .unwrap(); + assert_eq!(correct_metric.metric_value as i64, 0); + + // Verify per-step metrics were recorded for both rows. + let all_metrics = metrics_store.list_for_run(run_id).await.unwrap(); + let is_correct_count = all_metrics + .iter() + .filter(|m| m.metric_name == "is_correct") + .count(); + assert_eq!(is_correct_count, 2); + + // Verify steps were persisted in SQLite. + let steps = crate::db::list_steps_for_run(&db, run_id).await.unwrap(); + assert_eq!(steps.len(), 2); + + // Execute a second time to verify step caching reuses existing records. + let builtin2 = CustomNoCodeBuiltin::new(workflow_name.clone()); + builtin2 + .execute(super::BuiltinContext { + db: &db, + metrics_store: &metrics_store, + run_id, + workflow_name: &workflow_name, + input: Some(&input_json), + quiet: true, + }) + .await + .unwrap(); + + let steps2 = crate::db::list_steps_for_run(&db, run_id).await.unwrap(); + assert_eq!( + steps2.len(), + 2, + "second execution should reuse cached steps instead of creating new ones" + ); + + // Restore environment variables. + unsafe { + match &orig_hf { + Some(v) => std::env::set_var("HF_DATASETS_SERVER", v), + None => std::env::remove_var("HF_DATASETS_SERVER"), + } + match &orig_cache { + Some(v) => std::env::set_var("QUANTILES_DATASET_CACHE_DIR", v), + None => std::env::remove_var("QUANTILES_DATASET_CACHE_DIR"), + } + } + } } diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs index b8e5585..14a8510 100644 --- a/cli/src/commands/resume.rs +++ b/cli/src/commands/resume.rs @@ -213,4 +213,27 @@ mod tests { let err = plan_resume("my-eval", &RunStatus::Failed, Some(&bench)).unwrap_err(); assert!(err.to_string().contains("non-empty `command`")); } + + /// A `custom_nocode` benchmark should plan to resume as a builtin so that the + /// CLI can re-run the no-code workflow natively without spawning an external command. + #[test] + fn plan_resume_custom_nocode_with_config() { + let file = tempfile::NamedTempFile::new().unwrap(); + let bench = + qt::config::BenchmarkConfig::CustomNoCode(qt::config::CustomNoCodeBenchmarkConfig { + type_: "custom_nocode".to_owned(), + style: qt::config::CustomNoCodeStyle::Qa, + dataset: "quantiles/simpleqa-verified".to_owned(), + model: Some(qt::llm::Sampler::Random {}), + qa: qt::config::CustomNoCodeQaConfig { + prompt_template_file: file.path().to_str().unwrap().to_owned(), + prompt_column: "problem".to_owned(), + golden_column: "answer".to_owned(), + limit: None, + max_workers: None, + }, + }); + let plan = plan_resume("nocode_custom", &RunStatus::Failed, Some(&bench)).unwrap(); + assert!(matches!(plan, ResumePlan::Builtin)); + } } diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs index f8ffb04..d04ae15 100644 --- a/cli/src/commands/run.rs +++ b/cli/src/commands/run.rs @@ -738,4 +738,78 @@ mod tests { let (input, _) = super::assemble_builtin_input(None, None); assert!(input.is_none()); } + + /// A `custom_nocode` benchmark config with all fields should serialize into the + /// expected JSON shape, converting field names faithfully. + #[test] + fn assemble_custom_nocode_input_from_config() { + let bench = qt::config::CustomNoCodeBenchmarkConfig { + type_: "custom_nocode".to_owned(), + style: qt::config::CustomNoCodeStyle::Qa, + dataset: "quantiles/simpleqa-verified".to_owned(), + model: Some(qt::llm::Sampler::Random {}), + qa: qt::config::CustomNoCodeQaConfig { + prompt_template_file: "prompts/qa.txt".to_owned(), + prompt_column: "problem".to_owned(), + golden_column: "answer".to_owned(), + limit: Some(10), + max_workers: Some(4), + }, + }; + let input = super::assemble_custom_nocode_input(&bench, None); + let parsed: serde_json::Value = serde_json::from_str(&input).unwrap(); + assert_eq!(parsed["style"], "qa"); + assert_eq!(parsed["dataset"], "quantiles/simpleqa-verified"); + assert_eq!(parsed["model"], "random"); + assert_eq!(parsed["prompt_template_file"], "prompts/qa.txt"); + assert_eq!(parsed["prompt_column"], "problem"); + assert_eq!(parsed["golden_column"], "answer"); + assert_eq!(parsed["limit"], 10); + assert_eq!(parsed["max_workers"], 4); + } + + /// When `model`, `limit`, and `max_workers` are absent from the config, the assembled + /// JSON should omit those keys entirely rather than emit null values. + #[test] + fn assemble_custom_nocode_input_omits_none_fields() { + let bench = qt::config::CustomNoCodeBenchmarkConfig { + type_: "custom_nocode".to_owned(), + style: qt::config::CustomNoCodeStyle::Qa, + dataset: "quantiles/simpleqa-verified".to_owned(), + model: None, + qa: qt::config::CustomNoCodeQaConfig { + prompt_template_file: "prompts/qa.txt".to_owned(), + prompt_column: "problem".to_owned(), + golden_column: "answer".to_owned(), + limit: None, + max_workers: None, + }, + }; + let input = super::assemble_custom_nocode_input(&bench, None); + let parsed: serde_json::Value = serde_json::from_str(&input).unwrap(); + assert!(parsed.get("model").is_none()); + assert!(parsed.get("limit").is_none()); + assert!(parsed.get("max_workers").is_none()); + } + + /// A `--input` CLI flag should take precedence over the assembled config object, + /// returning the raw CLI string directly. + #[test] + fn assemble_custom_nocode_input_with_cli_override() { + let bench = qt::config::CustomNoCodeBenchmarkConfig { + type_: "custom_nocode".to_owned(), + style: qt::config::CustomNoCodeStyle::Qa, + dataset: "quantiles/simpleqa-verified".to_owned(), + model: None, + qa: qt::config::CustomNoCodeQaConfig { + prompt_template_file: "prompts/qa.txt".to_owned(), + prompt_column: "problem".to_owned(), + golden_column: "answer".to_owned(), + limit: None, + max_workers: None, + }, + }; + let input = super::assemble_custom_nocode_input(&bench, Some(r#"{"limit":5}"#)); + assert_eq!(input, r#"{"limit":5}"#); + } } diff --git a/cli/src/config.rs b/cli/src/config.rs index 0ecc773..306d785 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -363,6 +363,25 @@ mod tests { assert!(err.to_string().contains("not found")); } + #[test] + fn validate_accepts_existing_template_file() { + let file = tempfile::NamedTempFile::new().unwrap(); + let bench = BenchmarkConfig::CustomNoCode(CustomNoCodeBenchmarkConfig { + type_: "custom_nocode".to_owned(), + style: CustomNoCodeStyle::Qa, + dataset: "quantiles/simpleqa-verified".to_owned(), + model: Some(Sampler::Random), + qa: CustomNoCodeQaConfig { + prompt_template_file: file.path().to_str().unwrap().to_owned(), + prompt_column: "problem".to_owned(), + golden_column: "answer".to_owned(), + limit: None, + max_workers: None, + }, + }); + bench.validate().unwrap(); + } + #[test] fn validate_accepts_nonempty_command() { let bench = BenchmarkConfig::CustomCode(CustomCodeBenchmarkConfig { diff --git a/cli/src/dataset/hf_client.rs b/cli/src/dataset/hf_client.rs index 017ea41..37356d6 100644 --- a/cli/src/dataset/hf_client.rs +++ b/cli/src/dataset/hf_client.rs @@ -3,7 +3,10 @@ use reqwest::header; use serde::Deserialize; use serde_json::Value; -static HF_DATASETS_SERVER: &str = "https://datasets-server.huggingface.co"; +fn hf_base_url() -> String { + std::env::var("HF_DATASETS_SERVER") + .unwrap_or_else(|_| "https://datasets-server.huggingface.co".to_owned()) +} /// Thin async HTTP client for the huggingface Dataset Viewer API. pub struct HuggingFaceClient { @@ -104,7 +107,10 @@ impl HuggingFaceClient { config: &str, revision: Option<&str>, ) -> Result> { - let mut url = format!("{HF_DATASETS_SERVER}/splits?dataset={dataset_id}&config={config}"); + let mut url = format!( + "{}/splits?dataset={dataset_id}&config={config}", + hf_base_url() + ); if let Some(rev) = revision { url = format!("{url}&revision={rev}"); } @@ -127,7 +133,7 @@ impl HuggingFaceClient { /// Returns an error if configs couldn't be inferred from the given dataset /// and revision. pub async fn infer_config(&self, dataset_id: &str, revision: Option<&str>) -> Result { - let mut url = format!("{HF_DATASETS_SERVER}/splits?dataset={dataset_id}"); + let mut url = format!("{}/splits?dataset={dataset_id}", hf_base_url()); if let Some(rev) = revision { url = format!("{url}&revision={rev}"); } @@ -160,7 +166,8 @@ impl HuggingFaceClient { revision: Option<&str>, ) -> Result> { let mut url = format!( - "{HF_DATASETS_SERVER}/rows?dataset={dataset_id}&config={config}&split={split}&offset={offset}&length={limit}" + "{}/rows?dataset={dataset_id}&config={config}&split={split}&offset={offset}&length={limit}", + hf_base_url() ); if let Some(rev) = revision { url = format!("{url}&revision={rev}"); @@ -189,8 +196,10 @@ impl HuggingFaceClient { split: &str, revision: Option<&str>, ) -> Result { - let mut url = - format!("{HF_DATASETS_SERVER}/size?dataset={dataset_id}&config={config}&split={split}"); + let mut url = format!( + "{}/size?dataset={dataset_id}&config={config}&split={split}", + hf_base_url() + ); if let Some(rev) = revision { url = format!("{url}&revision={rev}"); } diff --git a/cli/src/dataset/mod.rs b/cli/src/dataset/mod.rs index b740fd8..fa78209 100644 --- a/cli/src/dataset/mod.rs +++ b/cli/src/dataset/mod.rs @@ -29,11 +29,15 @@ impl DatasetManager { /// /// Returns an error if the system's cache directory cannot be determined. pub fn new() -> Result { - let cache_dir = dirs::home_dir() - .context("failed to determine home directory")? - .join(".cache") - .join("quantiles") - .join("datasets"); + let cache_dir = if let Ok(dir) = std::env::var("QUANTILES_DATASET_CACHE_DIR") { + std::path::PathBuf::from(dir) + } else { + dirs::home_dir() + .context("failed to determine home directory")? + .join(".cache") + .join("quantiles") + .join("datasets") + }; let client = HuggingFaceClient::new()?; Ok(Self { client, From a0eda732318cdc66bf39c3f22a1a4379bdabc66a Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Fri, 3 Jul 2026 10:39:18 -0700 Subject: [PATCH 06/20] progress --- cli/src/builtins/common.rs | 52 +++++++++++++++++-- cli/src/builtins/custom_nocode.rs | 82 +++++++++-------------------- cli/src/builtins/mod.rs | 2 +- cli/src/builtins/pubmedqa/mod.rs | 41 +++------------ cli/src/builtins/similarity.rs | 10 ++-- cli/src/commands/resume.rs | 10 ++-- cli/src/commands/run.rs | 86 +++++++++++++++++-------------- 7 files changed, 138 insertions(+), 145 deletions(-) diff --git a/cli/src/builtins/common.rs b/cli/src/builtins/common.rs index 53ea370..e8b71e6 100644 --- a/cli/src/builtins/common.rs +++ b/cli/src/builtins/common.rs @@ -1,13 +1,15 @@ +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; +use std::time::Instant; + use anyhow::{Context, Result}; use sea_orm::DatabaseConnection; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; -use std::time::Instant; use crate::db::steps::{self, StepDecision}; -use crate::llm::Sampler; +use crate::llm::{LLMSampler, Sampler}; use crate::metrics_store::MetricsStore; /// Fields shared by every builtin benchmark config. When adding a new builtin, @@ -119,6 +121,48 @@ pub(crate) fn get_max_workers() -> usize { } } +/// Resolve a model sampler, falling back to a default when none is configured. +pub(crate) fn resolve_sampler( + model: Option<&Sampler>, + default: impl FnOnce() -> Arc, +) -> Result> { + match model { + None => Ok(default()), + Some(sampler) => sampler.resolve(), + } +} + +/// Emit aggregate `accuracy`, `correct_count`, and `total_count` metrics from a +/// collection of per-sample boolean correctness values. +#[expect(clippy::cast_precision_loss)] +pub(crate) async fn emit_accuracy_metrics( + metrics_store: &MetricsStore, + run_id: i64, + results: impl IntoIterator, +) { + let mut correct_count = 0usize; + let mut total_count = 0usize; + for is_correct in results { + total_count += 1; + if is_correct { + correct_count += 1; + } + } + + if total_count > 0 { + let accuracy = correct_count as f64 / total_count as f64; + metrics_store + .emit(run_id, None, "accuracy", accuracy, None) + .await; + metrics_store + .emit(run_id, None, "correct_count", correct_count as f64, None) + .await; + metrics_store + .emit(run_id, None, "total_count", total_count as f64, None) + .await; + } +} + /// Statistics computed from a collection of similarity scores. #[derive(Debug)] pub(crate) struct ScoreStatistics { diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index d95517d..eeed289 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -3,29 +3,26 @@ use std::sync::Arc; use anyhow::{Context, Result, bail}; use serde::{Deserialize, Serialize}; -use crate::builtins::common::{extract_text, get_max_workers, hash_input, run_timed_step}; +use crate::builtins::common::{ + emit_accuracy_metrics, extract_text, get_max_workers, hash_input, resolve_sampler, + run_timed_step, +}; use crate::builtins::dataset_runner::DatasetRunner; use crate::builtins::input::set_builtin_run_input; use crate::builtins::output::set_builtin_run_output; use crate::builtins::{BuiltinContext, BuiltinWorkflow}; use crate::dataset::DatasetManager; -use crate::llm::LLMSampler; use crate::llm::random::RandomSampler; /// Input deserialized from the JSON assembled by `commands::run`. -#[derive(Debug, Default, Deserialize)] +#[derive(Debug, Deserialize)] struct CustomNoCodeInput { - style: String, + style: crate::config::CustomNoCodeStyle, dataset: String, #[serde(default)] model: Option, - prompt_template_file: String, - prompt_column: String, - golden_column: String, - #[serde(default)] - limit: Option, - #[serde(default)] - max_workers: Option, + #[serde(flatten)] + qa: crate::config::CustomNoCodeQaConfig, } /// Per-row step output stored as JSON in the step record. @@ -63,25 +60,25 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { .map(serde_json::from_str) .transpose() .context("invalid builtin input JSON")? - .unwrap_or_default(); + .context("custom_nocode benchmark requires input configuration")?; - if config.style != "qa" { - bail!( - "unsupported custom_nocode style `{}`; only `qa` is supported", - config.style - ); + // The enum deserializer already rejects unknown styles; + // this match ensures the field is read and will become + // non-exhaustive when new variants are added. + match config.style { + crate::config::CustomNoCodeStyle::Qa => {} } - if config.limit == Some(0) { + if config.qa.limit == Some(0) { bail!("limit must be > 0"); } // Load template file and validate syntax. let template_str = - std::fs::read_to_string(&config.prompt_template_file).with_context(|| { + std::fs::read_to_string(&config.qa.prompt_template_file).with_context(|| { format!( "failed to read prompt template file `{}`", - config.prompt_template_file + config.qa.prompt_template_file ) })?; let env = jinja::Environment::new(); @@ -89,16 +86,13 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { .with_context(|| { format!( "invalid jinja syntax in prompt template file `{}`", - config.prompt_template_file + config.qa.prompt_template_file ) })?; - let max_workers = config.max_workers.unwrap_or_else(get_max_workers); + let max_workers = config.qa.max_workers.unwrap_or_else(get_max_workers); - let llm: Arc = match config.model { - None => Arc::new(RandomSampler::new(80)), - Some(ref sampler) => sampler.resolve()?, - }; + let llm = resolve_sampler(config.model.as_ref(), || Arc::new(RandomSampler::new(80)))?; let manager = DatasetManager::new()?; let info = manager.init(&config.dataset, None, None, None).await?; @@ -106,14 +100,14 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { let total = info .total_rows .context("could not determine dataset size; pass an explicit limit")?; - let limit = config.limit.unwrap_or(total).min(total); + let limit = config.qa.limit.unwrap_or(total).min(total); set_builtin_run_input( ctx.db, ctx.run_id, config.model.as_ref(), limit, - config.max_workers, + config.qa.max_workers, ) .await?; @@ -121,8 +115,8 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { let model = config.model.clone(); let run_id = ctx.run_id; let dataset = config.dataset.clone(); - let prompt_column = config.prompt_column.clone(); - let golden_column = config.golden_column.clone(); + let prompt_column = config.qa.prompt_column.clone(); + let golden_column = config.qa.golden_column.clone(); let template_str = Arc::new(template_str); let name = self.name(); @@ -199,34 +193,8 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { }) .await?; - let mut correct_count: usize = 0; let total_count = results.len(); - for is_correct in results { - if is_correct { - correct_count += 1; - } - } - - #[expect(clippy::cast_precision_loss)] - if total_count > 0 { - let accuracy = correct_count as f64 / total_count as f64; - - ctx.metrics_store - .emit(ctx.run_id, None, "accuracy", accuracy, None) - .await; - ctx.metrics_store - .emit( - ctx.run_id, - None, - "correct_count", - correct_count as f64, - None, - ) - .await; - ctx.metrics_store - .emit(ctx.run_id, None, "total_count", total_count as f64, None) - .await; - } + emit_accuracy_metrics(ctx.metrics_store, ctx.run_id, results).await; set_builtin_run_output(ctx.db, ctx.run_id, total_count).await?; diff --git a/cli/src/builtins/mod.rs b/cli/src/builtins/mod.rs index 3a364b4..2951f7a 100644 --- a/cli/src/builtins/mod.rs +++ b/cli/src/builtins/mod.rs @@ -30,7 +30,7 @@ pub struct BuiltinContext<'a> { /// Trait for builtin evals that run natively inside the CLI. #[async_trait] pub trait BuiltinWorkflow: Send + Sync { - /// Unique name of the builtin (e.g. "financebench"). + /// Unique name of the builtin (e.g. "simpleqa-verified", "financebench"). fn name(&self) -> String; /// Execute the builtin eval. async fn execute(&self, ctx: BuiltinContext<'_>) -> Result<()>; diff --git a/cli/src/builtins/pubmedqa/mod.rs b/cli/src/builtins/pubmedqa/mod.rs index 60242ad..4e0df77 100644 --- a/cli/src/builtins/pubmedqa/mod.rs +++ b/cli/src/builtins/pubmedqa/mod.rs @@ -2,13 +2,14 @@ use std::sync::Arc; use anyhow::{Context, Result, bail}; -use crate::builtins::common::{get_max_workers, hash_input, run_timed_step}; +use crate::builtins::common::{ + emit_accuracy_metrics, get_max_workers, hash_input, resolve_sampler, run_timed_step, +}; use crate::builtins::dataset_runner::DatasetRunner; use crate::builtins::input::set_builtin_run_input; use crate::builtins::output::set_builtin_run_output; use crate::builtins::{BuiltinContext, BuiltinWorkflow}; use crate::dataset::DatasetManager; -use crate::llm::LLMSampler; use crate::llm::random_label::RandomLabelSampler; use config::{PubMedQAConfig, RowOutput}; @@ -22,7 +23,6 @@ mod eval; /// `PubMedQA` builtin using the quantiles/PubMedQA dataset. pub struct PubmedqaBuiltin; -#[expect(clippy::too_many_lines)] #[async_trait::async_trait] impl BuiltinWorkflow for PubmedqaBuiltin { fn name(&self) -> String { @@ -43,10 +43,9 @@ impl BuiltinWorkflow for PubmedqaBuiltin { let max_workers = config.base.max_workers.unwrap_or_else(get_max_workers); - let llm: Arc = match config.base.model { - None => Arc::new(RandomLabelSampler::new(&["yes", "no", "maybe"])), - Some(ref sampler) => sampler.resolve()?, - }; + let llm = resolve_sampler(config.base.model.as_ref(), || { + Arc::new(RandomLabelSampler::new(&["yes", "no", "maybe"])) + })?; let manager = DatasetManager::new()?; let dataset_id = "quantiles/PubMedQA"; @@ -138,34 +137,8 @@ impl BuiltinWorkflow for PubmedqaBuiltin { }) .await?; - let mut correct_count: usize = 0; let total_count = results.len(); - for is_correct in results { - if is_correct { - correct_count += 1; - } - } - - #[expect(clippy::cast_precision_loss)] - if total_count > 0 { - let accuracy = correct_count as f64 / total_count as f64; - - ctx.metrics_store - .emit(ctx.run_id, None, "accuracy", accuracy, None) - .await; - ctx.metrics_store - .emit( - ctx.run_id, - None, - "correct_count", - correct_count as f64, - None, - ) - .await; - ctx.metrics_store - .emit(ctx.run_id, None, "total_count", total_count as f64, None) - .await; - } + emit_accuracy_metrics(ctx.metrics_store, ctx.run_id, results).await; set_builtin_run_output(ctx.db, ctx.run_id, total_count).await?; diff --git a/cli/src/builtins/similarity.rs b/cli/src/builtins/similarity.rs index f6d151a..1c8fe0f 100644 --- a/cli/src/builtins/similarity.rs +++ b/cli/src/builtins/similarity.rs @@ -3,14 +3,13 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use crate::builtins::common::{ - compute_statistics, extract_text, get_max_workers, hash_input, run_timed_step, + compute_statistics, extract_text, get_max_workers, hash_input, resolve_sampler, run_timed_step, }; use crate::builtins::dataset_runner::DatasetRunner; use crate::builtins::input::set_builtin_run_input; use crate::builtins::output::set_builtin_run_output; use crate::builtins::{BuiltinContext, BuiltinWorkflow}; use crate::dataset::DatasetManager; -use crate::llm::LLMSampler; use crate::llm::random::RandomSampler; use crate::similarity::{ SimilarityMetric, SimilarityMetricName, levenshtein::LevenshteinSimilarity, @@ -90,10 +89,9 @@ impl BuiltinWorkflow for SimilarityBenchmark { SimilarityMetricName::Cosine => Box::new(CosineSimilarity::try_new()?), }; - let llm: Arc = match config.base.model { - None => Arc::new(RandomSampler::new(80)), - Some(ref sampler) => sampler.resolve()?, - }; + let llm = resolve_sampler(config.base.model.as_ref(), || { + Arc::new(RandomSampler::new(80)) + })?; let manager = DatasetManager::new()?; let info = manager.init(self.dataset_id, None, None, None).await?; diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs index 14a8510..2cb79a6 100644 --- a/cli/src/commands/resume.rs +++ b/cli/src/commands/resume.rs @@ -103,16 +103,16 @@ pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<( _ => builtins::resolve(workflow_name) .with_context(|| format!("builtin `{workflow_name}` not found"))?, }; - super::run::execute_builtin( - &db, - &metrics_store, + super::run::execute_builtin(super::run::ExecuteBuiltinArgs { + db: &db, + metrics_store: &metrics_store, run_id, workflow_name, builtin, - stored_input, + input: stored_input, json, process_start, - ) + }) .await } ResumePlan::CustomCode(command) => { diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs index d04ae15..7be84aa 100644 --- a/cli/src/commands/run.rs +++ b/cli/src/commands/run.rs @@ -91,16 +91,16 @@ pub async fn run( let builtin = Box::new(qt::builtins::CustomNoCodeBuiltin::new( workflow_name.to_owned(), )); - execute_builtin( - &db, - &metrics_store, + execute_builtin(ExecuteBuiltinArgs { + db: &db, + metrics_store: &metrics_store, run_id, workflow_name, builtin, - Some(&input), + input: Some(&input), json, process_start, - ) + }) .await } } @@ -242,78 +242,88 @@ async fn run_builtin_workflow( println!("Created run {run_id}"); } - execute_builtin( - &db, - &metrics_store, + execute_builtin(ExecuteBuiltinArgs { + db: &db, + metrics_store: &metrics_store, run_id, workflow_name, builtin, input, json, process_start, - ) + }) .await } -#[expect(clippy::too_many_arguments)] -pub async fn execute_builtin( - db: &DatabaseConnection, - metrics_store: &MetricsStore, - run_id: i64, - workflow_name: &str, - builtin: Box, - input: Option<&str>, - json: bool, - process_start: Instant, -) -> Result<()> { - let builtin_result = builtin +/// Arguments for [`execute_builtin`]. +pub struct ExecuteBuiltinArgs<'a> { + pub db: &'a DatabaseConnection, + pub metrics_store: &'a MetricsStore, + pub run_id: i64, + pub workflow_name: &'a str, + pub builtin: Box, + pub input: Option<&'a str>, + pub json: bool, + pub process_start: Instant, +} + +pub async fn execute_builtin(args: ExecuteBuiltinArgs<'_>) -> Result<()> { + let builtin_result = args + .builtin .execute(builtins::BuiltinContext { - db, - metrics_store, - run_id, - workflow_name, - input, - quiet: json, + db: args.db, + metrics_store: args.metrics_store, + run_id: args.run_id, + workflow_name: args.workflow_name, + input: args.input, + quiet: args.json, }) .await; - let duration = process_start.elapsed(); + let duration = args.process_start.elapsed(); match &builtin_result { Ok(()) => { - db::complete_run(db, metrics_store, run_id).await?; - if !json { + db::complete_run(args.db, args.metrics_store, args.run_id).await?; + if !args.json { println!( - "Run {run_id} completed successfully in {:.2}s", + "Run {} completed successfully in {:.2}s", + args.run_id, duration.as_secs_f64() ); } } Err(err) => { let msg = format!("{err:#}"); - db::fail_run(db, metrics_store, run_id, &msg).await?; - if !json { - println!("Run {run_id} failed: {msg}"); + db::fail_run(args.db, args.metrics_store, args.run_id, &msg).await?; + if !args.json { + println!("Run {} failed: {msg}", args.run_id); } } } - let aggregate = metrics_store.list_aggregate_for_run(run_id).await?; + let aggregate = args + .metrics_store + .list_aggregate_for_run(args.run_id) + .await?; - if json { + if args.json { let metrics_map: HashMap = aggregate .iter() .map(|m| (m.metric_name.clone(), m.metric_value)) .collect(); let output = BuiltinRunJsonOutput { aggregate_metrics: metrics_map, - run_id, + run_id: args.run_id, warning: None, }; println!("{}", serde_json::to_string(&output)?); } else { print_aggregate_metrics_table(&aggregate); - println!("\nRun `qt show {run_id} --verbose` for sample-level details."); + println!( + "\nRun `qt show {} --verbose` for sample-level details.", + args.run_id + ); } builtin_result From 08f835113b9fd04e07835efe3f16635799190fce Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:04:24 -0700 Subject: [PATCH 07/20] progress --- cli/src/builtins/custom_nocode.rs | 216 +++++++++++++++------------ cli/src/config/custom_nocode.rs | 163 ++++++++++++++++++++ cli/src/{config.rs => config/mod.rs} | 154 +------------------ 3 files changed, 287 insertions(+), 246 deletions(-) create mode 100644 cli/src/config/custom_nocode.rs rename cli/src/{config.rs => config/mod.rs} (69%) diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index eeed289..d1140d9 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -45,6 +45,72 @@ impl CustomNoCodeBuiltin { pub fn new(name: String) -> Self { Self { name } } + + #[expect(clippy::too_many_arguments)] + async fn evaluate_row( + &self, + i: usize, + row: &serde_json::Value, + prompt_column: &str, + golden_column: &str, + template_str: &str, + env: &jinja::Environment<'_>, + model: Option<&crate::llm::Sampler>, + llm: &std::sync::Arc, + db: &sea_orm::DatabaseConnection, + metrics_store: &crate::metrics_store::MetricsStore, + run_id: i64, + ) -> Result { + let prompt = extract_text(row, prompt_column) + .with_context(|| format!("row {i}: missing prompt column `{prompt_column}`"))?; + let golden = extract_text(row, golden_column) + .with_context(|| format!("row {i}: missing golden column `{golden_column}`"))?; + + let rendered = env + .render_str(template_str, jinja::context!(prompt => &prompt)) + .with_context(|| format!("row {i}: failed to render prompt template"))?; + + let model_str = model + .as_ref() + .map_or("random".to_string(), std::string::ToString::to_string); + let input_hash = hash_input(&format!( + "{rendered}\nmodel={model_str}\nworkflow={}", + self.name() + )); + let step_key = format!("row-{i}"); + + let (output, step_id) = + run_timed_step(db, metrics_store, run_id, &step_key, &input_hash, async { + let model_response = llm + .sample(&rendered) + .await + .with_context(|| format!("failed to sample LLM for row {i}"))?; + + let is_correct = is_exact_match(&model_response, &golden); + + Ok(RowOutput { + input: rendered.clone(), + response: model_response, + golden, + is_correct, + }) + }) + .await?; + + if let Some(step_id) = step_id { + metrics_store + .emit( + run_id, + Some(step_id), + "is_correct", + if output.is_correct { 1.0 } else { 0.0 }, + None, + ) + .await; + } + + Ok(output.is_correct) + } } #[async_trait::async_trait] @@ -53,54 +119,17 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { self.name.clone() } - #[expect(clippy::too_many_lines)] async fn execute(&self, ctx: BuiltinContext<'_>) -> Result<()> { - let config: CustomNoCodeInput = ctx - .input - .map(serde_json::from_str) - .transpose() - .context("invalid builtin input JSON")? - .context("custom_nocode benchmark requires input configuration")?; - - // The enum deserializer already rejects unknown styles; - // this match ensures the field is read and will become - // non-exhaustive when new variants are added. - match config.style { - crate::config::CustomNoCodeStyle::Qa => {} - } + let config = parse_input(ctx.input)?; - if config.qa.limit == Some(0) { - bail!("limit must be > 0"); - } - - // Load template file and validate syntax. - let template_str = - std::fs::read_to_string(&config.qa.prompt_template_file).with_context(|| { - format!( - "failed to read prompt template file `{}`", - config.qa.prompt_template_file - ) - })?; - let env = jinja::Environment::new(); - env.render_str(&template_str, jinja::context!(prompt => "")) - .with_context(|| { - format!( - "invalid jinja syntax in prompt template file `{}`", - config.qa.prompt_template_file - ) - })?; + let (template_str, env) = load_template(&config.qa.prompt_template_file)?; let max_workers = config.qa.max_workers.unwrap_or_else(get_max_workers); let llm = resolve_sampler(config.model.as_ref(), || Arc::new(RandomSampler::new(80)))?; - let manager = DatasetManager::new()?; - let info = manager.init(&config.dataset, None, None, None).await?; - - let total = info - .total_rows - .context("could not determine dataset size; pass an explicit limit")?; - let limit = config.qa.limit.unwrap_or(total).min(total); + let (manager, info, limit) = + resolve_dataset_limit(&config.dataset, config.qa.limit).await?; set_builtin_run_input( ctx.db, @@ -132,63 +161,20 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { let golden_column = golden_column.clone(); let env = env.clone(); async move { - let prompt = extract_text(&row, &prompt_column).with_context(|| { - format!("row {i}: missing prompt column `{prompt_column}`") - })?; - let golden = extract_text(&row, &golden_column).with_context(|| { - format!("row {i}: missing golden column `{golden_column}`") - })?; - - let rendered = env - .render_str(&template_str, jinja::context!(prompt => &prompt)) - .with_context(|| format!("row {i}: failed to render prompt template"))?; - - let model_str = model - .as_ref() - .map_or("random".to_string(), std::string::ToString::to_string); - let input_hash = hash_input(&format!( - "{rendered}\nmodel={model_str}\nworkflow={}", - self.name() - )); - let step_key = format!("row-{i}"); - - let (output, step_id) = run_timed_step( + self.evaluate_row( + i, + &row, + &prompt_column, + &golden_column, + &template_str, + &env, + model.as_ref(), + &llm, &db, ctx.metrics_store, run_id, - &step_key, - &input_hash, - async { - let model_response = llm - .sample(&rendered) - .await - .with_context(|| format!("failed to sample LLM for row {i}"))?; - - let is_correct = is_exact_match(&model_response, &golden); - - Ok(RowOutput { - input: rendered.clone(), - response: model_response, - golden, - is_correct, - }) - }, ) - .await?; - - if let Some(step_id) = step_id { - ctx.metrics_store - .emit( - ctx.run_id, - Some(step_id), - "is_correct", - if output.is_correct { 1.0 } else { 0.0 }, - None, - ) - .await; - } - - Ok::<_, anyhow::Error>(output.is_correct) + .await } }) .await?; @@ -202,6 +188,48 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { } } +fn parse_input(input: Option<&str>) -> Result { + let config: CustomNoCodeInput = input + .map(serde_json::from_str) + .transpose() + .context("invalid builtin input JSON")? + .context("custom_nocode benchmark requires input configuration")?; + + match config.style { + crate::config::CustomNoCodeStyle::Qa => {} + } + + if config.qa.limit == Some(0) { + bail!("limit must be > 0"); + } + + Ok(config) +} + +fn load_template(path: &str) -> Result<(String, jinja::Environment<'_>)> { + let template_str = std::fs::read_to_string(path) + .with_context(|| format!("failed to read prompt template file `{path}`"))?; + let env = jinja::Environment::new(); + env.render_str(&template_str, jinja::context!(prompt => "")) + .with_context(|| format!("invalid jinja syntax in prompt template file `{path}`"))?; + Ok((template_str, env)) +} + +async fn resolve_dataset_limit( + dataset: &str, + limit: Option, +) -> Result<(DatasetManager, crate::dataset::DatasetInfo, usize)> { + let manager = DatasetManager::new()?; + let info = manager.init(dataset, None, None, None).await?; + + let total = info + .total_rows + .context("could not determine dataset size; pass an explicit limit")?; + let limit = limit.unwrap_or(total).min(total); + + Ok((manager, info, limit)) +} + /// Case-sensitive exact-match comparison after trimming whitespace. fn is_exact_match(response: &str, golden: &str) -> bool { response.trim() == golden.trim() diff --git a/cli/src/config/custom_nocode.rs b/cli/src/config/custom_nocode.rs new file mode 100644 index 0000000..0e3e3a9 --- /dev/null +++ b/cli/src/config/custom_nocode.rs @@ -0,0 +1,163 @@ +use anyhow::Result; +use serde::{Deserialize, Deserializer}; + +use crate::llm::Sampler; + +/// Style of a no-code custom benchmark. +/// +/// Future variants may include: +/// - `Judge` – evaluate responses against a rubric using a judge model. +/// - `Similarity` – score responses with a similarity metric. +/// - `Agent` – run multi-step agent loops. +#[derive(Debug, Clone)] +pub enum CustomNoCodeStyle { + Qa, +} + +impl<'de> Deserialize<'de> for CustomNoCodeStyle { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + match s.as_str() { + "qa" => Ok(CustomNoCodeStyle::Qa), + other => Err(serde::de::Error::custom(format!( + "unsupported custom_nocode style `{other}`; expected `qa`", + ))), + } + } +} + +/// QA-specific configuration for a no-code custom benchmark. +#[derive(Debug, Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct CustomNoCodeQaConfig { + /// Path to a Jinja template file for rendering prompts. + pub prompt_template_file: String, + /// Dataset column containing the prompt text. + pub prompt_column: String, + /// Dataset column containing the golden answer. + pub golden_column: String, + /// Number of dataset rows to evaluate. + pub limit: Option, + /// Maximum concurrent workers. + pub max_workers: Option, +} + +/// No-code custom benchmark configuration. +#[derive(Debug, Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct CustomNoCodeBenchmarkConfig { + #[serde(rename = "type")] + pub type_: String, + pub style: CustomNoCodeStyle, + /// Dataset identifier (e.g. `quantiles/simpleqa-verified`). + pub dataset: String, + /// Model sampler to use. + pub model: Option, + #[serde(flatten)] + pub qa: CustomNoCodeQaConfig, +} + +#[cfg(test)] +mod tests { + use super::super::{BenchmarkConfig, WorkspaceConfig}; + use super::*; + + #[test] + fn deserialize_custom_nocode_qa() { + let toml = r#" + [benchmarks.nocode_custom] + type = "custom_nocode" + style = "qa" + dataset = "quantiles/simpleqa-verified" + model = "random" + prompt_template_file = "prompts/qa.txt" + prompt_column = "problem" + golden_column = "answer" + limit = 10 + "#; + let config: WorkspaceConfig = toml::from_str(toml).unwrap(); + let bench = config.benchmarks.get("nocode_custom").unwrap(); + assert!(matches!(bench, BenchmarkConfig::CustomNoCode(_))); + if let BenchmarkConfig::CustomNoCode(c) = bench { + assert!(matches!(c.style, CustomNoCodeStyle::Qa)); + assert_eq!(c.dataset, "quantiles/simpleqa-verified"); + assert_eq!(c.model, Some(Sampler::Random)); + assert_eq!(c.qa.prompt_template_file, "prompts/qa.txt"); + assert_eq!(c.qa.prompt_column, "problem"); + assert_eq!(c.qa.golden_column, "answer"); + assert_eq!(c.qa.limit, Some(10)); + } + } + + #[test] + fn deserialize_custom_nocode_unsupported_style_errors() { + let toml = r#" + [benchmarks.nocode_custom] + type = "custom_nocode" + style = "judge" + dataset = "quantiles/simpleqa-verified" + model = "random" + prompt_template_file = "prompts/qa.txt" + prompt_column = "problem" + golden_column = "answer" + "#; + let result: Result = toml::from_str(toml); + assert!(result.is_err()); + } + + #[test] + fn custom_nocode_missing_required_field_errors() { + let toml = r#" + [benchmarks.nocode_custom] + type = "custom_nocode" + style = "qa" + dataset = "quantiles/simpleqa-verified" + model = "random" + prompt_column = "problem" + golden_column = "answer" + "#; + let result: Result = toml::from_str(toml); + assert!(result.is_err()); + } + + #[test] + fn validate_rejects_missing_template_file() { + let bench = BenchmarkConfig::CustomNoCode(CustomNoCodeBenchmarkConfig { + type_: "custom_nocode".to_owned(), + style: CustomNoCodeStyle::Qa, + dataset: "quantiles/simpleqa-verified".to_owned(), + model: Some(Sampler::Random), + qa: CustomNoCodeQaConfig { + prompt_template_file: "does_not_exist.txt".to_owned(), + prompt_column: "problem".to_owned(), + golden_column: "answer".to_owned(), + limit: None, + max_workers: None, + }, + }); + let err = bench.validate().unwrap_err(); + assert!(err.to_string().contains("not found")); + } + + #[test] + fn validate_accepts_existing_template_file() { + let file = tempfile::NamedTempFile::new().unwrap(); + let bench = BenchmarkConfig::CustomNoCode(CustomNoCodeBenchmarkConfig { + type_: "custom_nocode".to_owned(), + style: CustomNoCodeStyle::Qa, + dataset: "quantiles/simpleqa-verified".to_owned(), + model: Some(Sampler::Random), + qa: CustomNoCodeQaConfig { + prompt_template_file: file.path().to_str().unwrap().to_owned(), + prompt_column: "problem".to_owned(), + golden_column: "answer".to_owned(), + limit: None, + max_workers: None, + }, + }); + bench.validate().unwrap(); + } +} diff --git a/cli/src/config.rs b/cli/src/config/mod.rs similarity index 69% rename from cli/src/config.rs rename to cli/src/config/mod.rs index 306d785..b451ea3 100644 --- a/cli/src/config.rs +++ b/cli/src/config/mod.rs @@ -116,62 +116,8 @@ pub struct CustomCodeBenchmarkConfig { pub input: Option>, } -/// Style of a no-code custom benchmark. -/// -/// Future variants may include: -/// - `Judge` – evaluate responses against a rubric using a judge model. -/// - `Similarity` – score responses with a similarity metric. -/// - `Agent` – run multi-step agent loops. -#[derive(Debug, Clone)] -pub enum CustomNoCodeStyle { - Qa, -} - -impl<'de> Deserialize<'de> for CustomNoCodeStyle { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let s = String::deserialize(deserializer)?; - match s.as_str() { - "qa" => Ok(CustomNoCodeStyle::Qa), - other => Err(serde::de::Error::custom(format!( - "unsupported custom_nocode style `{other}`; expected `qa`", - ))), - } - } -} - -/// QA-specific configuration for a no-code custom benchmark. -#[derive(Debug, Deserialize, Clone)] -#[serde(deny_unknown_fields)] -pub struct CustomNoCodeQaConfig { - /// Path to a Jinja template file for rendering prompts. - pub prompt_template_file: String, - /// Dataset column containing the prompt text. - pub prompt_column: String, - /// Dataset column containing the golden answer. - pub golden_column: String, - /// Number of dataset rows to evaluate. - pub limit: Option, - /// Maximum concurrent workers. - pub max_workers: Option, -} - -/// No-code custom benchmark configuration. -#[derive(Debug, Deserialize, Clone)] -#[serde(deny_unknown_fields)] -pub struct CustomNoCodeBenchmarkConfig { - #[serde(rename = "type")] - pub type_: String, - pub style: CustomNoCodeStyle, - /// Dataset identifier (e.g. `quantiles/simpleqa-verified`). - pub dataset: String, - /// Model sampler to use. - pub model: Option, - #[serde(flatten)] - pub qa: CustomNoCodeQaConfig, -} +mod custom_nocode; +pub use custom_nocode::*; /// Top-level workspace configuration read from `quantiles.toml` or /// `.quantiles.toml` in the current working directory. @@ -344,44 +290,6 @@ mod tests { assert!(err.to_string().contains("non-empty `command`")); } - #[test] - fn validate_rejects_missing_template_file() { - let bench = BenchmarkConfig::CustomNoCode(CustomNoCodeBenchmarkConfig { - type_: "custom_nocode".to_owned(), - style: CustomNoCodeStyle::Qa, - dataset: "quantiles/simpleqa-verified".to_owned(), - model: Some(Sampler::Random), - qa: CustomNoCodeQaConfig { - prompt_template_file: "does_not_exist.txt".to_owned(), - prompt_column: "problem".to_owned(), - golden_column: "answer".to_owned(), - limit: None, - max_workers: None, - }, - }); - let err = bench.validate().unwrap_err(); - assert!(err.to_string().contains("not found")); - } - - #[test] - fn validate_accepts_existing_template_file() { - let file = tempfile::NamedTempFile::new().unwrap(); - let bench = BenchmarkConfig::CustomNoCode(CustomNoCodeBenchmarkConfig { - type_: "custom_nocode".to_owned(), - style: CustomNoCodeStyle::Qa, - dataset: "quantiles/simpleqa-verified".to_owned(), - model: Some(Sampler::Random), - qa: CustomNoCodeQaConfig { - prompt_template_file: file.path().to_str().unwrap().to_owned(), - prompt_column: "problem".to_owned(), - golden_column: "answer".to_owned(), - limit: None, - max_workers: None, - }, - }); - bench.validate().unwrap(); - } - #[test] fn validate_accepts_nonempty_command() { let bench = BenchmarkConfig::CustomCode(CustomCodeBenchmarkConfig { @@ -402,64 +310,6 @@ mod tests { assert!(result.is_err()); } - #[test] - fn deserialize_custom_nocode_qa() { - let toml = r#" - [benchmarks.nocode_custom] - type = "custom_nocode" - style = "qa" - dataset = "quantiles/simpleqa-verified" - model = "random" - prompt_template_file = "prompts/qa.txt" - prompt_column = "problem" - golden_column = "answer" - limit = 10 - "#; - let config: WorkspaceConfig = toml::from_str(toml).unwrap(); - let bench = config.benchmarks.get("nocode_custom").unwrap(); - assert!(matches!(bench, BenchmarkConfig::CustomNoCode(_))); - if let BenchmarkConfig::CustomNoCode(c) = bench { - assert!(matches!(c.style, CustomNoCodeStyle::Qa)); - assert_eq!(c.dataset, "quantiles/simpleqa-verified"); - assert_eq!(c.model, Some(Sampler::Random)); - assert_eq!(c.qa.prompt_template_file, "prompts/qa.txt"); - assert_eq!(c.qa.prompt_column, "problem"); - assert_eq!(c.qa.golden_column, "answer"); - assert_eq!(c.qa.limit, Some(10)); - } - } - - #[test] - fn deserialize_custom_nocode_unsupported_style_errors() { - let toml = r#" - [benchmarks.nocode_custom] - type = "custom_nocode" - style = "judge" - dataset = "quantiles/simpleqa-verified" - model = "random" - prompt_template_file = "prompts/qa.txt" - prompt_column = "problem" - golden_column = "answer" - "#; - let result: Result = toml::from_str(toml); - assert!(result.is_err()); - } - - #[test] - fn custom_nocode_missing_required_field_errors() { - let toml = r#" - [benchmarks.nocode_custom] - type = "custom_nocode" - style = "qa" - dataset = "quantiles/simpleqa-verified" - model = "random" - prompt_column = "problem" - golden_column = "answer" - "#; - let result: Result = toml::from_str(toml); - assert!(result.is_err()); - } - /// `custom_code` benchmarks must have a `command` field; omitting it should fail at /// parse time because the struct requires it. #[test] From 52d040dc59e23369a3c600a4e1491a3190f9ec1c Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:08:09 -0700 Subject: [PATCH 08/20] progress --- cli/src/commands/run.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs index 7be84aa..7bcd87c 100644 --- a/cli/src/commands/run.rs +++ b/cli/src/commands/run.rs @@ -255,7 +255,7 @@ async fn run_builtin_workflow( .await } -/// Arguments for [`execute_builtin`]. +/// Arguments for the [`execute_builtin`] function. pub struct ExecuteBuiltinArgs<'a> { pub db: &'a DatabaseConnection, pub metrics_store: &'a MetricsStore, From d8b68ce9839e6feb8aaa99463f2a0bb91530c5c4 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:10:04 -0700 Subject: [PATCH 09/20] doc --- cli/src/config/custom_nocode.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cli/src/config/custom_nocode.rs b/cli/src/config/custom_nocode.rs index 0e3e3a9..08899ef 100644 --- a/cli/src/config/custom_nocode.rs +++ b/cli/src/config/custom_nocode.rs @@ -11,6 +11,10 @@ use crate::llm::Sampler; /// - `Agent` – run multi-step agent loops. #[derive(Debug, Clone)] pub enum CustomNoCodeStyle { + /// A qa-style benchmark. These benchmarks generally have datasets with + /// a prompt and a golden answer, which the model under test must exactly + /// match (modulo minor whitespace and other cleanup) for it to "pass" + /// the sample Qa, } From 23b997b4e299e70f5127cd90c621ae564dc54ca8 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:11:01 -0700 Subject: [PATCH 10/20] progrss --- cli/src/config/custom_nocode.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/config/custom_nocode.rs b/cli/src/config/custom_nocode.rs index 08899ef..7b09411 100644 --- a/cli/src/config/custom_nocode.rs +++ b/cli/src/config/custom_nocode.rs @@ -56,7 +56,7 @@ pub struct CustomNoCodeBenchmarkConfig { #[serde(rename = "type")] pub type_: String, pub style: CustomNoCodeStyle, - /// Dataset identifier (e.g. `quantiles/simpleqa-verified`). + /// Dataset identifier in HuggingFace (e.g. `quantiles/simpleqa-verified`). pub dataset: String, /// Model sampler to use. pub model: Option, From 8b7545e31a58e6605e8886222ff1da3f816173e8 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:11:43 -0700 Subject: [PATCH 11/20] progress --- cli/src/config/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/config/mod.rs b/cli/src/config/mod.rs index b451ea3..9dd9dc6 100644 --- a/cli/src/config/mod.rs +++ b/cli/src/config/mod.rs @@ -34,7 +34,7 @@ impl BenchmarkConfig { BenchmarkConfig::CustomNoCode(c) => { if !std::path::Path::new(&c.qa.prompt_template_file).is_file() { bail!( - "custom_nocode benchmark config `prompt_template_file` must point to an existing file; `{}` not found", + "custom_nocode benchmark config `prompt_template_file` must point to an existing file. File `{}` was not found", c.qa.prompt_template_file ); } From 24eb51a4a619af42774a277a3a57c4b4ded59722 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:21:08 -0700 Subject: [PATCH 12/20] progrewss --- cli/src/builtins/custom_nocode.rs | 123 ++++++++++++++++++------------ cli/src/config/custom_nocode.rs | 2 +- 2 files changed, 77 insertions(+), 48 deletions(-) diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index d1140d9..2f2a38e 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -34,6 +34,32 @@ struct RowOutput { is_correct: bool, } +/// Arguments for the [`CustomNoCodeBuiltin::evaluate_row`] method +pub struct EvaluateRowArgs<'a> { + /// The row index + i: usize, + /// The row value + row: &'a serde_json::Value, + /// The name of the column in this row that has the prompt + prompt_column: &'a str, + /// The name of the column in this row that has the golden answer + golden_column: &'a str, + /// The prompt template + template_str: &'a str, + /// The pre-constructed jinja template env + env: &'a jinja::Environment<'a>, + /// The model to test + model: Option<&'a crate::llm::Sampler>, + /// TODO: figure out why we have 2 samplers! + llm: &'a std::sync::Arc, + /// The connection to the metadata DB + db: &'a sea_orm::DatabaseConnection, + /// Metrics storage + metrics_store: &'a crate::metrics_store::MetricsStore, + /// The run ID + run_id: i64, +} + /// No-code custom benchmark builtin. pub struct CustomNoCodeBuiltin { name: String, @@ -46,45 +72,47 @@ impl CustomNoCodeBuiltin { Self { name } } - #[expect(clippy::too_many_arguments)] - async fn evaluate_row( - &self, - i: usize, - row: &serde_json::Value, - prompt_column: &str, - golden_column: &str, - template_str: &str, - env: &jinja::Environment<'_>, - model: Option<&crate::llm::Sampler>, - llm: &std::sync::Arc, - db: &sea_orm::DatabaseConnection, - metrics_store: &crate::metrics_store::MetricsStore, - run_id: i64, - ) -> Result { - let prompt = extract_text(row, prompt_column) - .with_context(|| format!("row {i}: missing prompt column `{prompt_column}`"))?; - let golden = extract_text(row, golden_column) - .with_context(|| format!("row {i}: missing golden column `{golden_column}`"))?; - - let rendered = env - .render_str(template_str, jinja::context!(prompt => &prompt)) - .with_context(|| format!("row {i}: failed to render prompt template"))?; - - let model_str = model + async fn evaluate_row(&self, args: EvaluateRowArgs<'_>) -> Result { + let prompt = extract_text(args.row, args.prompt_column).with_context(|| { + format!( + "row {}: missing prompt column `{}`", + args.i, args.golden_column + ) + })?; + let golden = extract_text(args.row, args.golden_column).with_context(|| { + format!( + "row {}: missing golden column `{}`", + args.i, args.golden_column + ) + })?; + + let rendered = args + .env + .render_str(args.template_str, jinja::context!(prompt => &prompt)) + .with_context(|| format!("row {}: failed to render prompt template", args.i))?; + + let model_str = args + .model .as_ref() .map_or("random".to_string(), std::string::ToString::to_string); let input_hash = hash_input(&format!( "{rendered}\nmodel={model_str}\nworkflow={}", self.name() )); - let step_key = format!("row-{i}"); - - let (output, step_id) = - run_timed_step(db, metrics_store, run_id, &step_key, &input_hash, async { - let model_response = llm + let step_key = format!("row-{}", args.i); + + let (output, step_id) = run_timed_step( + args.db, + args.metrics_store, + args.run_id, + &step_key, + &input_hash, + async { + let model_response = args + .llm .sample(&rendered) .await - .with_context(|| format!("failed to sample LLM for row {i}"))?; + .with_context(|| format!("failed to sample LLM for row {}", args.i))?; let is_correct = is_exact_match(&model_response, &golden); @@ -94,13 +122,14 @@ impl CustomNoCodeBuiltin { golden, is_correct, }) - }) - .await?; + }, + ) + .await?; if let Some(step_id) = step_id { - metrics_store + args.metrics_store .emit( - run_id, + args.run_id, Some(step_id), "is_correct", if output.is_correct { 1.0 } else { 0.0 }, @@ -161,20 +190,20 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { let golden_column = golden_column.clone(); let env = env.clone(); async move { - self.evaluate_row( + let args = EvaluateRowArgs { i, - &row, - &prompt_column, - &golden_column, - &template_str, - &env, - model.as_ref(), - &llm, - &db, - ctx.metrics_store, + row: &row, + prompt_column: &prompt_column, + golden_column: &golden_column, + template_str: &template_str, + env: &env, + model: model.as_ref(), + llm: &llm, + db: &db, + metrics_store: ctx.metrics_store, run_id, - ) - .await + }; + self.evaluate_row(args).await } }) .await?; diff --git a/cli/src/config/custom_nocode.rs b/cli/src/config/custom_nocode.rs index 7b09411..7028208 100644 --- a/cli/src/config/custom_nocode.rs +++ b/cli/src/config/custom_nocode.rs @@ -56,7 +56,7 @@ pub struct CustomNoCodeBenchmarkConfig { #[serde(rename = "type")] pub type_: String, pub style: CustomNoCodeStyle, - /// Dataset identifier in HuggingFace (e.g. `quantiles/simpleqa-verified`). + /// Dataset identifier in `HuggingFace` (e.g. `quantiles/simpleqa-verified`). pub dataset: String, /// Model sampler to use. pub model: Option, From 8737f3fceb0dcba01416e04ef3bbd7501bd566a9 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:21:34 -0700 Subject: [PATCH 13/20] progress --- cli/src/builtins/custom_nocode.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index 2f2a38e..6d9e9d2 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -35,7 +35,7 @@ struct RowOutput { } /// Arguments for the [`CustomNoCodeBuiltin::evaluate_row`] method -pub struct EvaluateRowArgs<'a> { +struct EvaluateRowArgs<'a> { /// The row index i: usize, /// The row value From 075554ead01ae1660fa86351602f5628216e57d3 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:26:56 -0700 Subject: [PATCH 14/20] fixup --- cli/src/builtins/custom_nocode.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index 6d9e9d2..9b857cb 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -48,9 +48,9 @@ struct EvaluateRowArgs<'a> { template_str: &'a str, /// The pre-constructed jinja template env env: &'a jinja::Environment<'a>, - /// The model to test - model: Option<&'a crate::llm::Sampler>, - /// TODO: figure out why we have 2 samplers! + /// The name of the model we're sampling (used for cache key hashing only) + model_name: &'a str, + /// The actual model to sample llm: &'a std::sync::Arc, /// The connection to the metadata DB db: &'a sea_orm::DatabaseConnection, @@ -91,12 +91,9 @@ impl CustomNoCodeBuiltin { .render_str(args.template_str, jinja::context!(prompt => &prompt)) .with_context(|| format!("row {}: failed to render prompt template", args.i))?; - let model_str = args - .model - .as_ref() - .map_or("random".to_string(), std::string::ToString::to_string); let input_hash = hash_input(&format!( - "{rendered}\nmodel={model_str}\nworkflow={}", + "{rendered}\nmodel={}\nworkflow={}", + args.model_name, self.name() )); let step_key = format!("row-{}", args.i); @@ -170,7 +167,10 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { .await?; let db = ctx.db.clone(); - let model = config.model.clone(); + let model_name = config + .model + .as_ref() + .map_or("random".to_string(), std::string::ToString::to_string); let run_id = ctx.run_id; let dataset = config.dataset.clone(); let prompt_column = config.qa.prompt_column.clone(); @@ -184,7 +184,7 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { .for_each_concurrent(max_workers, move |i, row| { let llm = Arc::clone(&llm); let db = db.clone(); - let model = model.clone(); + let model_name = model_name.clone(); let template_str = Arc::clone(&template_str); let prompt_column = prompt_column.clone(); let golden_column = golden_column.clone(); @@ -197,7 +197,7 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { golden_column: &golden_column, template_str: &template_str, env: &env, - model: model.as_ref(), + model_name: &model_name, llm: &llm, db: &db, metrics_store: ctx.metrics_store, From eae0bd2c1a32f0505c5cbff660dda8d9d2d7ba7e Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:28:09 -0700 Subject: [PATCH 15/20] progress --- cli/src/builtins/custom_nocode.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cli/src/builtins/custom_nocode.rs b/cli/src/builtins/custom_nocode.rs index 9b857cb..3f1bf33 100644 --- a/cli/src/builtins/custom_nocode.rs +++ b/cli/src/builtins/custom_nocode.rs @@ -147,16 +147,12 @@ impl BuiltinWorkflow for CustomNoCodeBuiltin { async fn execute(&self, ctx: BuiltinContext<'_>) -> Result<()> { let config = parse_input(ctx.input)?; - let (template_str, env) = load_template(&config.qa.prompt_template_file)?; - let max_workers = config.qa.max_workers.unwrap_or_else(get_max_workers); - let llm = resolve_sampler(config.model.as_ref(), || Arc::new(RandomSampler::new(80)))?; let (manager, info, limit) = resolve_dataset_limit(&config.dataset, config.qa.limit).await?; - set_builtin_run_input( ctx.db, ctx.run_id, From 7dcad68c1d5609c21bb32a05a9b1ebe7c6054265 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:58:36 -0700 Subject: [PATCH 16/20] more tests --- cli/src/commands/compare/report.rs | 87 ++++++++++++++++++ cli/src/commands/resume.rs | 143 +++++++++++++++++++++++++++++ 2 files changed, 230 insertions(+) diff --git a/cli/src/commands/compare/report.rs b/cli/src/commands/compare/report.rs index 3520e72..e2a70b3 100644 --- a/cli/src/commands/compare/report.rs +++ b/cli/src/commands/compare/report.rs @@ -341,6 +341,93 @@ mod tests { assert!(json.get("warning").is_none()); } + /// Comparing two actual `custom_nocode` runs (differing accuracy, + /// identical `total_count`) should produce a `CompareReport` that correctly + /// flags the changed metric while keeping the unchanged one as "same". + #[tokio::test] + #[expect(clippy::items_after_statements)] + async fn compare_two_custom_nocode_runs() { + let tmpdir = tempfile::tempdir().unwrap(); + let root = tmpdir.path(); + + qt::db::init_workspace(root).await.unwrap(); + let db = qt::db::open_workspace(root).await.unwrap(); + let metrics_store = + qt::metrics_store::MetricsStore::new(qt::db::metrics_dir(root)).unwrap(); + + async fn seed_run( + db: &sea_orm::DatabaseConnection, + ms: &qt::metrics_store::MetricsStore, + accuracy: f64, + total_count: f64, + ) -> i64 { + let run_id = qt::db::create_run(db, "nocode_custom", None).await.unwrap(); + ms.emit(run_id, None, "accuracy", accuracy, None).await; + ms.emit(run_id, None, "total_count", total_count, None) + .await; + ms.flush(run_id).await.unwrap(); + qt::db::complete_run(db, ms, run_id).await.unwrap(); + run_id + } + + let run_a = seed_run(&db, &metrics_store, 0.5, 10.0).await; + let run_b = seed_run(&db, &metrics_store, 0.7, 10.0).await; + + let data_a = RunData::fetch_by_id(&db, &metrics_store, run_a) + .await + .unwrap(); + let data_b = RunData::fetch_by_id(&db, &metrics_store, run_b) + .await + .unwrap(); + + let report = CompareReport::build(data_a, data_b); + + // The report should detect differences. + assert!(report.differs, "report should flag differing metrics"); + + // Both metrics should appear in the comparison. + assert_eq!(report.metrics.len(), 2, "expected 2 metrics compared"); + + // Accuracy differs between the two runs. + let accuracy_cmp = report + .metrics + .iter() + .find(|m| m.name == "accuracy") + .expect("accuracy metric should be present"); + assert!(!accuracy_cmp.same, "accuracy should differ"); + + // Total count is identical. + let total_cmp = report + .metrics + .iter() + .find(|m| m.name == "total_count") + .expect("total_count metric should be present"); + assert!(total_cmp.same, "total_count should be the same"); + + // Use JSON to verify the underlying values without touching private fields. + let json = serde_json::to_value(&report).expect("report should serialize"); + let metrics_json = json["metrics"].as_array().unwrap(); + let accuracy_json = metrics_json + .iter() + .find(|m| m["name"] == "accuracy") + .unwrap(); + assert_eq!(accuracy_json["run_a"], serde_json::json!([0.5])); + assert_eq!(accuracy_json["run_b"], serde_json::json!([0.7])); + + let total_json = metrics_json + .iter() + .find(|m| m["name"] == "total_count") + .unwrap(); + assert_eq!(total_json["run_a"], serde_json::json!([10.0])); + assert_eq!(total_json["run_b"], serde_json::json!([10.0])); + + // No benchmark-name warning since both runs share the same workflow. + assert!( + report.warning.is_none(), + "no warning expected for identical benchmark names" + ); + } + fn run_data(id: i64, workflow_name: &str) -> RunData { let now = OffsetDateTime::now_utc(); RunData { diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs index 2cb79a6..027e287 100644 --- a/cli/src/commands/resume.rs +++ b/cli/src/commands/resume.rs @@ -236,4 +236,147 @@ mod tests { let plan = plan_resume("nocode_custom", &RunStatus::Failed, Some(&bench)).unwrap(); assert!(matches!(plan, ResumePlan::Builtin)); } + + /// A failed `custom_nocode` run can be resumed and re-execute successfully + /// through the `CustomNoCodeBuiltin`, verifying that the resume path wires + /// the correct builtin and `ExecuteBuiltinArgs`. + #[expect(clippy::too_many_lines)] + #[tokio::test] + async fn resume_failed_custom_nocode_run_re_executes_successfully() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + let tmpdir = tempfile::tempdir().unwrap(); + let root = tmpdir.path(); + let cache_dir = root.join("cache"); + + // Save original env var values so we can restore them later. + let orig_hf = std::env::var("HF_DATASETS_SERVER").ok(); + let orig_cache = std::env::var("QUANTILES_DATASET_CACHE_DIR").ok(); + unsafe { + std::env::set_var("HF_DATASETS_SERVER", server.uri()); + std::env::set_var("QUANTILES_DATASET_CACHE_DIR", cache_dir.as_os_str()); + } + + // Mock HF dataset server endpoints used by DatasetManager::init(). + Mock::given(method("GET")) + .and(path("/splits")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "splits": [{"config": "default", "split": "train"}] + }))) + .mount(&server) + .await; + + Mock::given(method("GET")) + .and(path("/size")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "size": {"splits": [{"num_rows": 2}]} + }))) + .mount(&server) + .await; + + // Initialize workspace with SQLite DB and metrics dir. + qt::db::init_workspace(root).await.unwrap(); + let db = qt::db::open_workspace(root).await.unwrap(); + let metrics_store = + qt::metrics_store::MetricsStore::new(qt::db::metrics_dir(root)).unwrap(); + + // Write a Jinja template file. + let template_path = root.join("template.txt"); + std::fs::write(&template_path, "{{ prompt }}\nAnswer:").unwrap(); + + // Pre-populate the dataset cache so no network fetch is needed for rows. + let cache = qt::dataset::cache::DatasetCache::new(cache_dir); + let rows = vec![ + serde_json::json!({"question": "what is 2+2", "answer": "4"}), + serde_json::json!({"question": "what is 3+3", "answer": "6"}), + ]; + let key = qt::dataset::cache::cache_key("fixture/qa", "default", "train", None); + let batch_path = cache.batch_path(&key, 0, 2); + cache.write_batch(&batch_path, &rows).await.unwrap(); + + // Write a quantiles.toml so resume can load config. + std::fs::write( + root.join("quantiles.toml"), + format!( + r#" +[benchmarks.nocode_resume_test] +type = "custom_nocode" +style = "qa" +dataset = "fixture/qa" +model = "random" +prompt_template_file = "{}" +prompt_column = "question" +golden_column = "answer" +limit = 2 +"#, + template_path.to_str().unwrap() + ), + ) + .unwrap(); + + let input_json = serde_json::to_string(&serde_json::json!({ + "style": "qa", + "dataset": "fixture/qa", + "model": "random", + "prompt_template_file": template_path.to_str().unwrap(), + "prompt_column": "question", + "golden_column": "answer", + "limit": 2, + })) + .unwrap(); + + let workflow_name = "nocode_resume_test"; + let run_id = qt::db::create_run(&db, workflow_name, Some(&input_json)) + .await + .unwrap(); + + // Mark the run as failed so it can be resumed. + qt::db::fail_run(&db, &metrics_store, run_id, "simulated failure") + .await + .unwrap(); + + let run_before = qt::db::get_run(&db, run_id).await.unwrap(); + assert_eq!(run_before.status, qt::db::RunStatus::Failed); + + // Change cwd so resume finds the config file. + let orig_cwd = std::env::current_dir().unwrap(); + std::env::set_current_dir(root).unwrap(); + + resume(run_id, true, std::time::Instant::now()) + .await + .unwrap(); + + // Restore cwd. + std::env::set_current_dir(orig_cwd).unwrap(); + + // Verify the run is now completed. + let run_after = qt::db::get_run(&db, run_id).await.unwrap(); + assert_eq!(run_after.status, qt::db::RunStatus::Completed); + + // Verify aggregate metrics were written. + metrics_store.flush(run_id).await.unwrap(); + let agg = metrics_store.list_aggregate_for_run(run_id).await.unwrap(); + let names: Vec<&str> = agg.iter().map(|m| m.metric_name.as_str()).collect(); + assert!(names.contains(&"accuracy")); + assert!(names.contains(&"correct_count")); + assert!(names.contains(&"total_count")); + + // Verify steps were persisted in SQLite. + let steps = qt::db::list_steps_for_run(&db, run_id).await.unwrap(); + assert_eq!(steps.len(), 2); + + // Restore environment variables. + unsafe { + match &orig_hf { + Some(v) => std::env::set_var("HF_DATASETS_SERVER", v), + None => std::env::remove_var("HF_DATASETS_SERVER"), + } + match &orig_cache { + Some(v) => std::env::set_var("QUANTILES_DATASET_CACHE_DIR", v), + None => std::env::remove_var("QUANTILES_DATASET_CACHE_DIR"), + } + } + } } From 68f722416e07004a6275c22f1b61615855d90177 Mon Sep 17 00:00:00 2001 From: phranzia <9207473+phranzia@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:20:52 -0700 Subject: [PATCH 17/20] Update docs for no-code qa --- CONFIG.md | 41 +++++++++++++++++++++++++++++++++++++-- README.md | 8 +++++++- cli/README.md | 24 ++++++++++++++++++++++- docs/assets/new-badge.svg | 5 +++++ 4 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 docs/assets/new-badge.svg diff --git a/CONFIG.md b/CONFIG.md index ec62c4e..a5968f6 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -8,6 +8,7 @@ You need a config file when you want to do one or more of the following: - Override built-in benchmark defaults (e.g. model, sample limit) - Define custom evaluations (`type = "custom_code"`) +- Define no-code QA benchmarks (`type = "custom_nocode"`, `style = "qa"`) - Resume custom evaluations later with `qt resume ` You don't, however, need any configuration when you want to run built-in benchmarks. `qt run pubmedqa`, for example, works out of the box. @@ -54,7 +55,7 @@ command = ["uv", "run", "my_eval.py"] ## Benchmark types -Every benchmark section has a `type` field. Valid values are `"builtin"` (default when absent) and `"custom_code"`. +Every benchmark section has a `type` field. Valid values are `"builtin"` (default when absent), `"custom_code"`, and `"custom_nocode"`. ### `builtin` @@ -64,7 +65,7 @@ Built-in benchmarks run natively inside the CLI, without any custom code. Below |--|--|--|--| | `type` | string | no | Defaults to `"builtin"`. May be omitted for built-in benchmarks. | | `samples` | integer | no | Number of dataset rows to evaluate. | -| `model` | string or table | no | Model sampler. See [model format](#model-format). | +| `model` | string or table | no | Model sampler. See [model naming](#model-naming). | | `max_workers` | integer | no | Maximum concurrent workers. | If none of these fields are customized, the built-in benchmark uses the following defaults: @@ -97,6 +98,42 @@ model = { provider = "openai", model_id = "gpt-5.4-nano" } Note that models require specific configuration based on the provider. For details, see the `quantiles.toml` file under the provider of your choice in [`cli/examples/configs`](./cli/examples/configs). +### `custom_nocode` + +No-code benchmarks are configured in TOML and run natively inside the CLI, without a custom Python or TypeScript evaluation program. The initial supported style is `qa`, which renders a prompt from a dataset row, calls a model, and scores the response with exact-match accuracy against the configured golden answer column. + +```toml +[benchmarks.nocode_custom] +type = "custom_nocode" +style = "qa" +dataset = "quantiles/simpleqa-verified" +model = "random" +prompt_template_file = "prompts/qa.txt" +prompt_column = "problem" +golden_column = "answer" +limit = 10 +``` + +Run it with: + +```bash +qt run nocode_custom +``` + +For a complete minimal example, see [`custom-nocode-examples/quantiles.toml`](./custom-nocode-examples/quantiles.toml). + +| Field | Type | Required | Description | +|--|--|--|--| +| `type` | string | yes | Must be `"custom_nocode"`. | +| `style` | string | yes | Must be `"qa"`. | +| `dataset` | string | yes | Dataset identifier, for example `"quantiles/simpleqa-verified"`. | +| `model` | string or table | no | Model sampler. Defaults to the demo random sampler. See [model naming](#model-naming). | +| `prompt_template_file` | string | yes | Path to a Jinja prompt template file. The template receives `prompt` from the configured prompt column. | +| `prompt_column` | string | yes | Dataset column containing the prompt text. | +| `golden_column` | string | yes | Dataset column containing the golden answer. | +| `limit` | integer | no | Number of dataset rows to evaluate. | +| `max_workers` | integer | no | Maximum concurrent workers. | + ### `custom_code` Custom evaluations are external programs built with the Quantiles Python SDK. Their config sections contain the following fields: diff --git a/README.md b/README.md index de27d70..e67fab4 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ Quantiles is a local-first CLI and SDK for running durable AI evaluation workflo It includes a CLI (called `qt`), SDKs, built-in benchmarks, local run history, and agent-friendly instructions for coding agents such as Codex, Claude Code, Cursor, GitHub Copilot, Gemini CLI, OpenCode, and other agentic development tools. This monorepo centralizes all the pieces of Quantiles, making it easier for engineers and coding agents to inspect, change, test, and extend the system. +## ![NEW!](./docs/assets/new-badge.svg) What's New + +**[2026.07.07]** Added `custom_nocode` QA benchmarks, which let users configure a dataset-backed question-answering benchmark in `quantiles.toml` without writing custom evaluation code. See [No-code QA benchmarks](./cli/src/builtins/custom_nocode.rs) for more details. + ## Why Quantiles Evaluation workflows quickly outgrow one-off scripts once teams need caching, retries, dataset handling, metrics capture, and run comparison. Quantiles gives teams those primitives so they don't have to built them from scratch: @@ -117,7 +121,9 @@ Both the CLI and Python SDK support offline benchmark workflows, including the f ## Built-in Benchmarks -Built-in benchmarks are ready-to-run evalulations with predefined datasets, scoring methodologies, and metrics. Use them when you want a standardized evaluation that provides a common reference point, a repeatable baseline, or a well-defined implementation of an industry benchmark. +Built-in benchmarks are ready-to-run evaluations with predefined datasets, scoring methodologies, and metrics. Use them when you want a standardized evaluation that provides a common reference point, a repeatable baseline, or a well-defined implementation of an industry benchmark. + +For dataset-backed QA checks that do not need custom Python or TypeScript code, use a `custom_nocode` benchmark in `quantiles.toml`. A `custom_nocode` QA benchmark points to a dataset, a Jinja prompt template, the dataset column containing the prompt, and the dataset column containing the golden answer. See [`custom-nocode-examples/quantiles.toml`](./custom-nocode-examples/quantiles.toml) for a minimal example. | Code | When to use | | --- | --- | diff --git a/cli/README.md b/cli/README.md index eb556f5..2ce5c51 100644 --- a/cli/README.md +++ b/cli/README.md @@ -53,7 +53,7 @@ See [`examples/configs/custom_code/quantiles.toml`](./examples/configs/custom_co ## Configuration files and customization -You can customize how the CLI executes built-in benchmarks and custom evaluations using a `quantiles.toml` or `.quantiles.toml` configuration file. See the following resources for information and examples: +You can customize how the CLI executes built-in benchmarks, custom code evaluations, and no-code QA benchmarks using a `quantiles.toml` or `.quantiles.toml` configuration file. See the following resources for information and examples: - [`../CONFIG.md`](../CONFIG.md): for a guide and reference. - [`./examples/configs`](./examples/configs) for complete working examples. @@ -71,6 +71,28 @@ max_workers = 100 >Note: Quantiles is designed for high-throughput execution and may issue many requests in parallel. Depending on your provider, model, and account limits, benchmark runs can quickly hit API rate limits or concurrency quotas. Consider reducing concurrency or using models/providers with higher rate limits if you encounter throttling. Example configurations illustrate how to do so. +### No-code QA benchmarks + +For dataset-backed QA checks that do not need custom evaluation code, set `type = "custom_nocode"` and `style = "qa"`. The benchmark runs inside the CLI, renders each prompt with the configured Jinja template, calls the configured model, and scores each row with exact-match accuracy against the golden answer column. + +```toml +[benchmarks.nocode_custom] +type = "custom_nocode" +style = "qa" +dataset = "quantiles/simpleqa-verified" +model = "random" +prompt_template_file = "prompts/qa.txt" +prompt_column = "problem" +golden_column = "answer" +limit = 10 +``` + +```bash +qt run nocode_custom +``` + +See [`../custom-nocode-examples/quantiles.toml`](../custom-nocode-examples/quantiles.toml) for a complete minimal example. + ### Custom code evals For custom evaluations, set `type = "custom_code"` and provide the `command` to run. The optional `input` table is passed to your script as a JSON dictionary. diff --git a/docs/assets/new-badge.svg b/docs/assets/new-badge.svg new file mode 100644 index 0000000..07e5906 --- /dev/null +++ b/docs/assets/new-badge.svg @@ -0,0 +1,5 @@ + + NEW! + + NEW! + From b4b5f5eb96c304901fdbbaa355f38de35169a968 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 10:34:07 -0700 Subject: [PATCH 18/20] adding more tests --- cli/src/builtins/common.rs | 80 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/cli/src/builtins/common.rs b/cli/src/builtins/common.rs index e8b71e6..8b119c1 100644 --- a/cli/src/builtins/common.rs +++ b/cli/src/builtins/common.rs @@ -324,4 +324,84 @@ mod tests { assert!(stats.min <= stats.median); assert!(stats.median <= stats.max); } + + #[rstest] + fn test_resolve_sampler_uses_default_when_none() { + let result = resolve_sampler(None, || Arc::new(crate::llm::random::RandomSampler::new(80))) + .unwrap(); + // Should get the default sampler, not panic or error. + assert!(!result.sample("test").block_on().is_empty()); + } + + #[rstest] + fn test_resolve_sampler_resolves_configured_sampler() { + let sampler = crate::llm::Sampler::Random {}; + let result = resolve_sampler(Some(&sampler), || { + panic!("default should not be called when model is Some") + }) + .unwrap(); + // Should get a resolved sampler, not the default. + assert!(!result.sample("test").block_on().is_empty()); + } + + #[rstest] + fn test_emit_accuracy_metrics_empty() { + let tmpdir = tempfile::tempdir().unwrap(); + let metrics_store = crate::metrics_store::MetricsStore::new(tmpdir.path().to_path_buf()) + .unwrap(); + + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + emit_accuracy_metrics(&metrics_store, 1, [false; 0]).await; + metrics_store.flush(1).await.unwrap(); + let agg = metrics_store.list_aggregate_for_run(1).await.unwrap(); + assert!(agg.is_empty(), "no metrics should be emitted for empty results"); + }); + } + + #[rstest] + fn test_emit_accuracy_metrics_all_correct() { + let tmpdir = tempfile::tempdir().unwrap(); + let metrics_store = crate::metrics_store::MetricsStore::new(tmpdir.path().to_path_buf()) + .unwrap(); + + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + emit_accuracy_metrics(&metrics_store, 1, [true, true, true]).await; + metrics_store.flush(1).await.unwrap(); + let agg = metrics_store.list_aggregate_for_run(1).await.unwrap(); + + let accuracy = agg.iter().find(|m| m.metric_name == "accuracy").unwrap(); + assert!((accuracy.metric_value - 1.0).abs() < 1e-10); + + let correct = agg.iter().find(|m| m.metric_name == "correct_count").unwrap(); + assert_eq!(correct.metric_value as i64, 3); + + let total = agg.iter().find(|m| m.metric_name == "total_count").unwrap(); + assert_eq!(total.metric_value as i64, 3); + }); + } + + #[rstest] + fn test_emit_accuracy_metrics_mixed() { + let tmpdir = tempfile::tempdir().unwrap(); + let metrics_store = crate::metrics_store::MetricsStore::new(tmpdir.path().to_path_buf()) + .unwrap(); + + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + emit_accuracy_metrics(&metrics_store, 1, [true, false, true, false]).await; + metrics_store.flush(1).await.unwrap(); + let agg = metrics_store.list_aggregate_for_run(1).await.unwrap(); + + let accuracy = agg.iter().find(|m| m.metric_name == "accuracy").unwrap(); + assert!((accuracy.metric_value - 0.5).abs() < 1e-10); + + let correct = agg.iter().find(|m| m.metric_name == "correct_count").unwrap(); + assert_eq!(correct.metric_value as i64, 2); + + let total = agg.iter().find(|m| m.metric_name == "total_count").unwrap(); + assert_eq!(total.metric_value as i64, 4); + }); + } } From 98b19a848201671d291dab675fb4e8684e275dac Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:11:35 -0700 Subject: [PATCH 19/20] Adding GitHub workflow for building and publishing the CLI --- .github/workflows/cli-release.yml | 92 +++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 .github/workflows/cli-release.yml diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml new file mode 100644 index 0000000..5ee6af7 --- /dev/null +++ b/.github/workflows/cli-release.yml @@ -0,0 +1,92 @@ +name: cli-release + +on: + push: + tags: ["v*"] + workflow_dispatch: + +permissions: + contents: read + +env: + QT_R2_BUCKET: ${{ vars.QT_R2_BUCKET || 'quantiles-cli' }} + +jobs: + build: + name: ${{ matrix.target }} + strategy: + fail-fast: false + matrix: + include: + - runner: macos-13 + target: x86_64-apple-darwin + - runner: macos-14 + target: aarch64-apple-darwin + - runner: ubuntu-24.04 + target: x86_64-unknown-linux-gnu + - runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + runs-on: ${{ matrix.runner }} + defaults: + run: + working-directory: cli + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + - uses: Swatinem/rust-cache@v2 + with: + workspaces: cli + - name: Build release binary + run: cargo build --locked --release --target ${{ matrix.target }} + - name: Package release artifact + env: + TARGET: ${{ matrix.target }} + run: | + version="$(cargo metadata --locked --no-deps --format-version 1 | jq -r '.packages[0].version')" + archive="qt-${version}-${TARGET}.tar.gz" + mkdir -p dist/package + cp "target/${TARGET}/release/qt" dist/package/qt + COPYFILE_DISABLE=1 tar -czf "dist/${archive}" -C dist/package qt + shasum -a 256 "dist/${archive}" > "dist/${archive}.sha256" + - uses: actions/upload-artifact@v4 + with: + name: cli-${{ matrix.target }} + path: cli/dist/qt-*.tar.gz* + if-no-files-found: error + + publish: + name: publish to R2 + needs: build + runs-on: ubuntu-24.04 + environment: release + env: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: cli-* + path: dist + merge-multiple: true + - uses: dtolnay/rust-toolchain@stable + - name: Install Wrangler + run: npm install --global wrangler@4 + - name: Publish release artifacts and installer + working-directory: cli + run: | + version="$(cargo metadata --locked --no-deps --format-version 1 | jq -r '.packages[0].version')" + if [[ "$GITHUB_REF_TYPE" == "tag" && "$GITHUB_REF_NAME" != "v${version}" ]]; then + echo "tag $GITHUB_REF_NAME does not match CLI version v${version}" >&2 + exit 1 + fi + object_prefix="releases/v${version}" + + for file in ../dist/qt-*.tar.gz ../dist/qt-*.tar.gz.sha256; do + wrangler r2 object put "${QT_R2_BUCKET}/${object_prefix}/$(basename "$file")" --file "$file" --remote --cache-control "no-store" + done + + wrangler r2 object put "${QT_R2_BUCKET}/install.sh" --file scripts/install.sh --remote --cache-control "no-store" + wrangler r2 object put "${QT_R2_BUCKET}/${object_prefix}/install.sh" --file scripts/install.sh --remote --cache-control "no-store" From 2621a472522776be76c09f2982a71609f721c729 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:14:30 -0700 Subject: [PATCH 20/20] progress --- cli/mise.toml | 11 +++----- cli/scripts/upload-install-script-r2.sh | 34 ------------------------- 2 files changed, 3 insertions(+), 42 deletions(-) delete mode 100755 cli/scripts/upload-install-script-r2.sh diff --git a/cli/mise.toml b/cli/mise.toml index 5b9cdca..120f6bb 100644 --- a/cli/mise.toml +++ b/cli/mise.toml @@ -40,15 +40,10 @@ run = [ [tasks.run-http-client-example] run="cargo run --example http_client" -# build the CLI for ARM Mac, package the binaries, and publish the archives, -# checksums, and install script to the quantiles-cli R2 bucket +# dispatch the GitHub Actions release workflow, which builds the CLI for all +# supported macOS and Linux targets and publishes the release artifacts to R2 [tasks.publish-cli-r2] -run="bash scripts/publish-cli-r2.sh" - -# upload only the install script to R2; use this when installer logic changes -# but the already-published CLI release artifact does not need to be rebuilt -[tasks.upload-install-script-r2] -run="bash scripts/upload-install-script-r2.sh" +run="gh workflow run cli-release.yml" # lint shell scripts [tasks.shellcheck] diff --git a/cli/scripts/upload-install-script-r2.sh b/cli/scripts/upload-install-script-r2.sh deleted file mode 100755 index dc8d2d1..0000000 --- a/cli/scripts/upload-install-script-r2.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -bucket="${QT_R2_BUCKET:-quantiles-cli}" -dist_dir="${QT_DIST_DIR:-dist}" - -if ! command -v wrangler >/dev/null 2>&1; then - echo "wrangler is required to publish to Cloudflare R2" >&2 - exit 1 -fi - -if ! command -v jq >/dev/null 2>&1; then - echo "jq is required to parse cargo metadata" >&2 - exit 1 -fi - -version="$(cargo metadata --no-deps --format-version 1 | jq -r '.packages[0].version')" -if [[ -z "$version" || "$version" == "null" ]]; then - echo "failed to read crate version from cargo metadata" >&2 - exit 1 -fi - -mkdir -p "$dist_dir" -install_script="${dist_dir}/install.sh" -object_prefix="releases/v${version}" - -cp scripts/install.sh "$install_script" -chmod +x "$install_script" - -wrangler r2 object put "${bucket}/install.sh" --file "$install_script" --remote --cache-control "no-store" -wrangler r2 object put "${bucket}/${object_prefix}/install.sh" --file "$install_script" --remote --cache-control "no-store" - -echo "published installer to r2://${bucket}/install.sh" -echo "published versioned installer to r2://${bucket}/${object_prefix}/install.sh"