From 5a3e8eccb558571b977ab1ad931c58e964e9422e Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 11:21:17 -0700
Subject: [PATCH 01/26] Adding `qt resume` command

---
 .../configs/custom_code/quantiles.toml        |  26 ++
 cli/src/cli.rs                                |  12 +-
 cli/src/commands/mod.rs                       |   2 +
 cli/src/commands/resume.rs                    |  91 ++++
 cli/src/commands/run.rs                       | 391 +++++++++++-------
 cli/src/config.rs                             |  61 ++-
 cli/src/main.rs                               |  15 +-
 cli/src/metrics_store.rs                      |   4 +-
 8 files changed, 429 insertions(+), 173 deletions(-)
 create mode 100644 cli/examples/configs/custom_code/quantiles.toml
 create mode 100644 cli/src/commands/resume.rs

diff --git a/cli/examples/configs/custom_code/quantiles.toml b/cli/examples/configs/custom_code/quantiles.toml
new file mode 100644
index 0000000..4bf1ab6
--- /dev/null
+++ b/cli/examples/configs/custom_code/quantiles.toml
@@ -0,0 +1,26 @@
+# This config file demonstrates running a custom evaluation with `qt run`.
+#
+# Custom benchmarks use `type = "custom_code"` and must specify a `command`
+# that the CLI will execute. The `input` table is optional and its keys are
+# passed to the child process via the `QUANTILES_INPUT` environment variable
+# as a JSON object.
+#
+# Usage from this directory:
+#
+#   qt run my-eval
+#
+# This will run the command below, injecting `QUANTILES_RUN_ID`,
+# `QUANTILES_WORKFLOW_NAME`, `QUANTILES_BASE_URL`, and `QUANTILES_INPUT`
+# into the child environment.
+#
+# If the run fails, you can resume it later with only the run ID:
+#
+#   qt resume <run_id>
+#
+# The CLI will look up the stored workflow name and input from the database,
+# then re-read the command from this config file.
+
+[benchmarks.my-eval]
+type = "custom_code"
+command = ["python", "my_eval.py"]
+input = {foo = "foo_val", bar = "bar_val"}
diff --git a/cli/src/cli.rs b/cli/src/cli.rs
index 1c0fb69..8bae80c 100644
--- a/cli/src/cli.rs
+++ b/cli/src/cli.rs
@@ -33,16 +33,16 @@ pub enum Command {
         workflow_name: String,
         #[arg(long)]
         input: Option<String>,
-        /// Resume an existing eval run instead of creating a new one.
+        /// Emit machine-readable JSON.
         #[arg(long)]
-        resume: Option<i64>,
+        json: bool,
+    },
+    /// Resume a running or failed eval run.
+    Resume {
+        run_id: i64,
         /// Emit machine-readable JSON.
         #[arg(long)]
         json: bool,
-        /// The command and arguments to execute.
-        /// Optional when running a builtin eval.
-        #[arg(last = true, required = false)]
-        command: Vec<String>,
     },
     /// Start the local qt HTTP server. Mostly used for local development or custom setups.
     Serve {
diff --git a/cli/src/commands/mod.rs b/cli/src/commands/mod.rs
index c117c6e..138f35e 100644
--- a/cli/src/commands/mod.rs
+++ b/cli/src/commands/mod.rs
@@ -1,6 +1,7 @@
 mod compare;
 mod init;
 mod list;
+mod resume;
 mod run;
 mod serve;
 mod show;
@@ -8,6 +9,7 @@ mod show;
 pub use compare::compare;
 pub use init::init;
 pub use list::list;
+pub use resume::resume;
 pub use run::run;
 pub use serve::serve;
 pub use show::show;
diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs
new file mode 100644
index 0000000..32f3027
--- /dev/null
+++ b/cli/src/commands/resume.rs
@@ -0,0 +1,91 @@
+use std::time::Instant;
+
+use anyhow::{Result, bail};
+
+use qt::builtins;
+use qt::db;
+use qt::metrics_store::MetricsStore;
+
+/// Resume an existing eval run.
+///
+/// # Errors
+///
+/// Returns an error when the run does not exist, is already completed, the
+/// config file is missing or invalid, or execution fails.
+pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<()> {
+    let cwd = std::env::current_dir()?;
+    let root = db::resolve_workspace_root(&cwd, true).await?;
+    let db = db::open_workspace(&root).await?;
+    let metrics_store = MetricsStore::new(db::metrics_dir(&root))?;
+
+    let run = db::get_run(&db, run_id).await?;
+    if run.status == db::RunStatus::Completed {
+        bail!(
+            "run {run_id} is already completed; \
+             create a new run or resume a running/failed one"
+        );
+    }
+
+    let workflow_name = run.workflow_name.as_str();
+    let stored_input = run.input.as_deref();
+
+    let config = qt::config::load()?;
+    let bench_config = config.benchmarks.get(workflow_name);
+
+    db::resume_run(&db, run_id).await?;
+    if !json {
+        println!("Resuming eval run {run_id} ({workflow_name})");
+    }
+
+    match bench_config {
+        Some(bench) => {
+            bench.validate()?;
+            match bench.type_ {
+                qt::config::BenchmarkType::Builtin => {
+                    super::run::execute_builtin(
+                        &db,
+                        &metrics_store,
+                        run_id,
+                        workflow_name,
+                        stored_input,
+                        json,
+                        process_start,
+                    )
+                    .await
+                }
+                qt::config::BenchmarkType::CustomCode => {
+                    let command = bench.command.as_ref().unwrap();
+                    super::run::execute_custom(
+                        run_id,
+                        workflow_name,
+                        stored_input,
+                        command,
+                        json,
+                        process_start,
+                        None,
+                    )
+                    .await
+                }
+            }
+        }
+        None => {
+            if builtins::resolve(workflow_name).is_some() {
+                super::run::execute_builtin(
+                    &db,
+                    &metrics_store,
+                    run_id,
+                    workflow_name,
+                    stored_input,
+                    json,
+                    process_start,
+                )
+                .await
+            } else {
+                bail!(
+                    "no config section found for benchmark `{workflow_name}`; \
+                     cannot resume custom eval without config"
+                );
+            }
+        }
+    }
+}
diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs
index 0d54fec..440e90d 100644
--- a/cli/src/commands/run.rs
+++ b/cli/src/commands/run.rs
@@ -7,6 +7,7 @@ use std::time::{Duration, Instant};
 
 use anyhow::{Context, Result, bail};
 use comfy_table::{Cell, ContentArrangement, Table, presets::NOTHING};
+use sea_orm::DatabaseConnection;
 use serde::Serialize;
 
 use qt::builtins;
@@ -16,25 +17,250 @@ use qt::db::MetricPointSummary;
 use qt::metrics_store::MetricsStore;
 use qt::server::{self, ServerConfig};
 
-#[expect(clippy::too_many_lines)]
 pub async fn run(
     workflow_name: &str,
-    input: Option<&str>,
-    resume: Option<i64>,
+    cli_input: Option<&str>,
     json: bool,
-    command: &[String],
     process_start: Instant,
 ) -> Result<()> {
-    if command.is_empty() {
-        if builtins::resolve(workflow_name).is_some() {
-            return run_builtin(workflow_name, input, resume, json, process_start).await;
+    let config = qt::config::load()?;
+
+    let bench_config = config.benchmarks.get(workflow_name);
+
+    match bench_config {
+        Some(bench) => {
+            bench.validate()?;
+            match bench.type_ {
+                qt::config::BenchmarkType::Builtin => {
+                    let (effective_input, _) = assemble_builtin_input(Some(bench), cli_input);
+                    run_builtin_workflow(
+                        workflow_name,
+                        effective_input.as_deref(),
+                        json,
+                        process_start,
+                    )
+                    .await
+                }
+                qt::config::BenchmarkType::CustomCode => {
+                    let (merged_input, overridden_keys) =
+                        merge_inputs(bench.input.as_ref(), cli_input)?;
+                    let warning = if overridden_keys.is_empty() {
+                        None
+                    } else {
+                        Some(format!(
+                            "--input overrides config input for keys: {}",
+                            overridden_keys.join(", ")
+                        ))
+                    };
+                    let command = bench.command.as_ref().unwrap();
+
+                    let cwd = std::env::current_dir()?;
+                    let root = db::resolve_workspace_root(&cwd, true).await?;
+                    let db = db::open_workspace(&root).await?;
+                    let run_id =
+                        db::create_run(&db, workflow_name, merged_input.as_deref()).await?;
+
+                    if !json {
+                        println!("Created run {run_id}");
+                    }
+
+                    execute_custom(
+                        run_id,
+                        workflow_name,
+                        merged_input.as_deref(),
+                        command,
+                        json,
+                        process_start,
+                        warning.as_deref(),
+                    )
+                    .await
+                }
+            }
+        }
+        None => {
+            if builtins::resolve(workflow_name).is_some() {
+                let (effective_input, _) = assemble_builtin_input(None, cli_input);
+                run_builtin_workflow(
+                    workflow_name,
+                    effective_input.as_deref(),
+                    json,
+                    process_start,
+                )
+                .await
+            } else {
+                bail!("no config section found for benchmark `{workflow_name}`");
+            }
+        }
+    }
+}
+
+fn assemble_builtin_input(
+    bench: Option<&qt::config::BenchmarkConfig>,
+    cli_input: Option<&str>,
+) -> (Option<String>, Vec<String>) {
+    if let Some(cli_str) = cli_input {
+        return (Some(cli_str.to_owned()), Vec::new());
+    }
+
+    if let Some(bench) = bench {
+        if bench.samples.is_none() && bench.model.is_none() && bench.max_workers.is_none() {
+            return (None, Vec::new());
         }
-        bail!("no command provided and `{workflow_name}` is not a builtin eval");
+
+        let input = BuiltinConfigInput {
+            limit: bench.samples,
+            model: bench.model.clone(),
+            max_workers: bench.max_workers,
+        };
+
+        let json =
+            serde_json::to_string(&input).expect("infallible serialization of BuiltinConfigInput");
+
+        (Some(json), Vec::new())
+    } else {
+        (None, Vec::new())
     }
+}
 
+fn merge_inputs(
+    config_input: Option<&serde_json::Value>,
+    cli_input: Option<&str>,
+) -> Result<(Option<String>, Vec<String>)> {
+    let mut merged = match config_input {
+        Some(v) if v.is_object() => v.clone(),
+        Some(_) => {
+            bail!("config input is not a JSON object");
+        }
+        None => serde_json::Value::Object(serde_json::Map::default()),
+    };
+
+    let mut overridden = Vec::new();
+
+    if let Some(cli_str) = cli_input {
+        let cli_json: serde_json::Value =
+            serde_json::from_str(cli_str).with_context(|| "failed to parse --input as JSON")?;
+
+        if let Some(cli_obj) = cli_json.as_object()
+            && let Some(merged_obj) = merged.as_object_mut()
+        {
+            for (key, value) in cli_obj {
+                if merged_obj.contains_key(key) {
+                    overridden.push(key.clone());
+                }
+                merged_obj.insert(key.clone(), value.clone());
+            }
+        }
+    }
+
+    if merged.as_object().is_some_and(serde_json::Map::is_empty) {
+        Ok((None, overridden))
+    } else {
+        Ok((Some(serde_json::to_string(&merged)?), overridden))
+    }
+}
+
+async fn run_builtin_workflow(
+    workflow_name: &str,
+    input: Option<&str>,
+    json: bool,
+    process_start: Instant,
+) -> Result<()> {
     let cwd = std::env::current_dir()?;
     let root = db::resolve_workspace_root(&cwd, true).await?;
+    let db = db::open_workspace(&root).await?;
+    let metrics_store = MetricsStore::new(db::metrics_dir(&root))?;
+
+    let run_id = db::create_run(&db, workflow_name, input).await?;
+    if !json {
+        println!("Created run {run_id}");
+    }
+
+    execute_builtin(
+        &db,
+        &metrics_store,
+        run_id,
+        workflow_name,
+        input,
+        json,
+        process_start,
+    )
+    .await
+}
+
+pub async fn execute_builtin(
+    db: &DatabaseConnection,
+    metrics_store: &MetricsStore,
+    run_id: i64,
+    workflow_name: &str,
+    input: Option<&str>,
+    json: bool,
+    process_start: Instant,
+) -> Result<()> {
+    let builtin = builtins::resolve(workflow_name)
+        .with_context(|| format!("builtin `{workflow_name}` not found"))?;
+
+    let builtin_result = builtin
+        .execute(builtins::BuiltinContext {
+            db,
+            metrics_store,
+            run_id,
+            workflow_name,
+            input,
+            quiet: json,
+        })
+        .await;
+
+    let duration = process_start.elapsed();
+
+    match &builtin_result {
+        Ok(()) => {
+            db::complete_run(db, metrics_store, run_id).await?;
+            if !json {
+                println!(
+                    "Run {run_id} completed successfully in {:.2}s",
+                    duration.as_secs_f64()
+                );
+            }
+        }
+        Err(err) => {
+            let msg = format!("{err:#}");
+            db::fail_run(db, metrics_store, run_id, &msg).await?;
+            if !json {
+                println!("Run {run_id} failed: {msg}");
+            }
+        }
+    }
+
+    let aggregate = metrics_store.list_aggregate_for_run(run_id).await?;
+
+    if json {
+        let metrics_map: HashMap<String, f64> = aggregate
+            .iter()
+            .map(|m| (m.metric_name.clone(), m.metric_value))
+            .collect();
+        let output = BuiltinRunJsonOutput {
+            aggregate_metrics: metrics_map,
+            run_id,
+            warning: None,
+        };
+        println!("{}", serde_json::to_string(&output)?);
+    } else {
+        print_aggregate_metrics_table(&aggregate);
+        println!("\nRun `qt show {run_id} --verbose` for sample-level details.");
+    }
+
+    builtin_result
+}
 
+pub async fn execute_custom(
+    run_id: i64,
+    workflow_name: &str,
+    input: Option<&str>,
+    command: &[String],
+    json: bool,
+    process_start: Instant,
+    warning: Option<&str>,
+) -> Result<()> {
     let base_url =
         std::env::var("QUANTILES_BASE_URL").unwrap_or_else(|_| "http://127.0.0.1:8765".to_owned());
     let addr = base_url
@@ -43,6 +269,8 @@ pub async fn run(
 
     let server_started_by_us = !is_server_reachable(addr);
     let server_handle = if server_started_by_us {
+        let cwd = std::env::current_dir()?;
+        let root = db::resolve_workspace_root(&cwd, true).await?;
         let db_path = db::workspace_db_path(&root);
         Some(start_background_server(addr, json, db_path)?)
     } else {
@@ -51,29 +279,6 @@ pub async fn run(
 
     wait_for_health(&base_url)?;
 
-    let db = db::open_workspace(&root).await?;
-
-    let run_id = if let Some(existing_run_id) = resume {
-        let run = db::get_run(&db, existing_run_id).await?;
-        if run.status == db::RunStatus::Completed {
-            bail!(
-                "run {existing_run_id} is already completed; \
-                 create a new run or resume a running/failed one"
-            );
-        }
-        db::resume_run(&db, existing_run_id).await?;
-        if !json {
-            println!("Resuming eval run {existing_run_id} ({workflow_name})");
-        }
-        existing_run_id
-    } else {
-        let new_run_id = db::create_run(&db, workflow_name, input).await?;
-        if !json {
-            println!("Created run {new_run_id}");
-        }
-        new_run_id
-    };
-
     if !json {
         println!("Executing: {}", command.join(" "));
     }
@@ -123,7 +328,7 @@ pub async fn run(
             run_id,
             workflow_name,
             input,
-            resumed: resume.is_some(),
+            warning,
             command,
             base_url: &base_url,
             server_started_by_us,
@@ -141,121 +346,6 @@ pub async fn run(
     Ok(())
 }
 
-async fn run_builtin(
-    workflow_name: &str,
-    input: Option<&str>,
-    resume: Option<i64>,
-    json: bool,
-    process_start: Instant,
-) -> Result<()> {
-    let cwd = std::env::current_dir()?;
-    let root = db::resolve_workspace_root(&cwd, true).await?;
-
-    let config = qt::config::load()?;
-    let config_input: Option<String> = if input.is_none() {
-        config.benchmarks.get(workflow_name).and_then(|bench| {
-            if bench.samples.is_none() && bench.model.is_none() && bench.max_workers.is_none() {
-                return None;
-            }
-            let input = BuiltinConfigInput {
-                limit: bench.samples,
-                model: bench.model.clone(),
-                max_workers: bench.max_workers,
-            };
-            Some(
-                serde_json::to_string(&input)
-                    .expect("infallible serialization of BuiltinConfigInput"),
-            )
-        })
-    } else {
-        None
-    };
-
-    let effective_input: Option<&str> = if input.is_some() {
-        input
-    } else {
-        config_input.as_deref()
-    };
-
-    let db = db::open_workspace(&root).await?;
-    let metrics_store = MetricsStore::new(db::metrics_dir(&root))?;
-
-    let run_id = if let Some(existing_run_id) = resume {
-        let run = db::get_run(&db, existing_run_id).await?;
-        if run.status == db::RunStatus::Completed {
-            bail!(
-                "run {existing_run_id} is already completed; \
-                 create a new run or resume a running/failed one"
-            );
-        }
-        db::resume_run(&db, existing_run_id).await?;
-        if !json {
-            println!("Resuming eval run {existing_run_id} ({workflow_name})");
-        }
-        existing_run_id
-    } else {
-        let new_run_id = db::create_run(&db, workflow_name, effective_input).await?;
-        if !json {
-            println!("Created run {new_run_id}");
-        }
-        new_run_id
-    };
-
-    let builtin = builtins::resolve(workflow_name)
-        .with_context(|| format!("builtin `{workflow_name}` not found"))?;
-
-    let builtin_result = builtin
-        .execute(builtins::BuiltinContext {
-            db: &db,
-            metrics_store: &metrics_store,
-            run_id,
-            workflow_name,
-            input: effective_input,
-            quiet: json,
-        })
-        .await;
-
-    let duration = process_start.elapsed();
-
-    match &builtin_result {
-        Ok(()) => {
-            db::complete_run(&db, &metrics_store, run_id).await?;
-            if !json {
-                println!(
-                    "Run {run_id} completed successfully in {:.2}s",
-                    duration.as_secs_f64()
-                );
-            }
-        }
-        Err(err) => {
-            let msg = format!("{err:#}");
-            db::fail_run(&db, &metrics_store, run_id, &msg).await?;
-            if !json {
-                println!("Run {run_id} failed: {msg}");
-            }
-        }
-    }
-
-    let aggregate = metrics_store.list_aggregate_for_run(run_id).await?;
-
-    if json {
-        let metrics_map: HashMap<String, f64> = aggregate
-            .iter()
-            .map(|m| (m.metric_name.clone(), m.metric_value))
-            .collect();
-        let output = BuiltinRunJsonOutput {
-            aggregate_metrics: metrics_map,
-            run_id,
-        };
-        println!("{}", serde_json::to_string(&output)?);
-    } else {
-        print_aggregate_metrics_table(&aggregate);
-        println!("\nRun `qt show {run_id} --verbose` for sample-level details.");
-    }
-
-    builtin_result
-}
-
 fn print_aggregate_metrics_table(metrics: &[MetricPointSummary]) {
     if metrics.is_empty() {
         println!("\nAggregate metrics");
@@ -286,6 +376,8 @@ fn print_aggregate_metrics_table(metrics: &[MetricPointSummary]) {
 struct BuiltinRunJsonOutput {
     aggregate_metrics: HashMap<String, f64>,
     run_id: i64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    warning: Option<String>,
 }
 
 /// Config input shape auto-generated from `quantiles.toml` `[benchmarks.*]`.
@@ -318,8 +410,9 @@ struct RunJsonOutput<'a> {
     workflow_name: &'a str,
     /// Structured run input, if one was provided.
     input: Option<&'a str>,
-    /// Whether this command resumed an existing run.
-    resumed: bool,
+    /// Warning about overridden config input keys.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    warning: Option<&'a str>,
     /// User command and arguments that were executed.
     command: &'a [String],
     /// Local qt API base URL injected into the child process.
diff --git a/cli/src/config.rs b/cli/src/config.rs
index 64fe5ec..c429257 100644
--- a/cli/src/config.rs
+++ b/cli/src/config.rs
@@ -5,22 +5,75 @@ use serde::Deserialize;
 
 use crate::llm::Sampler;
 
+/// Type of benchmark execution.
+#[derive(Debug, Default, Deserialize, Clone, PartialEq)]
+#[serde(rename_all = "snake_case")]
+pub enum BenchmarkType {
+    /// Native builtin eval implemented in the CLI.
+    #[default]
+    Builtin,
+    /// External custom eval run as a child process.
+    CustomCode,
+}
+
 /// Configuration for a single benchmark.
 #[derive(Debug, Deserialize, Default)]
 pub struct BenchmarkConfig {
-    /// Number of samples (rows) to evaluate.
+    /// Execution type. Defaults to `builtin`.
+    #[serde(default, rename = "type")]
+    pub type_: BenchmarkType,
+    /// Number of samples (rows) to evaluate. (builtin only)
     pub samples: Option<usize>,
-    /// Which model sampler to use for this benchmark.
+    /// Which model sampler to use for this benchmark. (builtin only)
     pub model: Option<Sampler>,
-    /// Maximum concurrent workers for this benchmark.
+    /// Maximum concurrent workers for this benchmark. (builtin only)
     pub max_workers: Option<usize>,
+    /// Command and arguments to execute. (`custom_code` only)
+    pub command: Option<Vec<String>>,
+    /// Structured input object passed to the eval. (`custom_code` only)
+    pub input: Option<serde_json::Value>,
+}
+
+impl BenchmarkConfig {
+    /// Validate that the benchmark config fields are consistent with its `type`.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error when a required field is missing or a disallowed field is present.
+    pub fn validate(&self) -> Result<()> {
+        match self.type_ {
+            BenchmarkType::Builtin => {
+                if self.command.is_some() {
+                    bail!("builtin benchmark config cannot have a `command` field");
+                }
+                if self.input.is_some() {
+                    bail!("builtin benchmark config cannot have an `input` field");
+                }
+            }
+            BenchmarkType::CustomCode => {
+                if self.command.as_ref().is_none_or(Vec::is_empty) {
+                    bail!(
+                        "custom_code benchmark config must have a non-empty `command` field"
+                    );
+                }
+                if let Some(ref input) = self.input
+                    && !input.is_object()
+                {
+                    bail!(
+                        "custom_code benchmark config `input` must be a JSON object / TOML table"
+                    );
+                }
+            }
+        }
+        Ok(())
+    }
 }
 
 /// Top-level workspace configuration read from `quantiles.toml` or
 /// `.quantiles.toml` in the current working directory.
 #[derive(Debug, Deserialize, Default)]
 pub struct WorkspaceConfig {
-    /// Per-benchmark overrides keyed by the builtin workflow name.
+    /// Per-benchmark overrides keyed by the workflow name.
     #[serde(default)]
     pub benchmarks: HashMap<String, BenchmarkConfig>,
 }
diff --git a/cli/src/main.rs b/cli/src/main.rs
index 64181ec..176c68b 100644
--- a/cli/src/main.rs
+++ b/cli/src/main.rs
@@ -28,19 +28,10 @@ async fn async_main(process_start: Instant) -> Result<()> {
         cli::Command::Run {
             workflow_name,
             input,
-            resume,
             json,
-            command,
-        } => {
-            commands::run(
-                &workflow_name,
-                input.as_deref(),
-                resume,
-                json,
-                &command,
-                process_start,
-            )
-            .await
+        } => commands::run(&workflow_name, input.as_deref(), json, process_start).await,
+        cli::Command::Resume { run_id, json } => {
+            commands::resume(run_id, json, process_start).await
         }
         cli::Command::Serve { addr } => commands::serve(&addr).await,
         cli::Command::Show { run_id, json } => commands::show(run_id, json).await,
diff --git a/cli/src/metrics_store.rs b/cli/src/metrics_store.rs
index dc4cc22..2a5a9f1 100644
--- a/cli/src/metrics_store.rs
+++ b/cli/src/metrics_store.rs
@@ -35,11 +35,11 @@ pub struct MetricsStore {
     ///
     /// TODO: use a concurrent queue rather than sequential hashmap,
     ///
-    /// https://docs.rs/crossbeam-queue/latest/crossbeam_queue/struct.SegQueue.html
+    /// <https://docs.rs/crossbeam-queue/latest/crossbeam_queue/struct.SegQueue.html>
     ///
     /// This is a work in progress at:
     ///
-    /// https://github.com/quantiles-evals/quantiles/pull/7
+    /// <https://github.com/quantiles-evals/quantiles/pull/7>
     buffer: Arc<Mutex<HashMap<i64, Vec<BufferedMetric>>>>,
     /// The directory inside which metrics are saved.
     dir: PathBuf,

From 36fbe402ee3ce945818c5e8d17dd309c9d8c7920 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 11:26:47 -0700
Subject: [PATCH 02/26] JSON flag TODO

---
 cli/src/commands/resume.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs
index 32f3027..1758ec3 100644
--- a/cli/src/commands/resume.rs
+++ b/cli/src/commands/resume.rs
@@ -20,6 +20,7 @@ pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<(
 
     let run = db::get_run(&db, run_id).await?;
     if run.status == db::RunStatus::Completed {
+        // TODO: json flag here
         bail!(
             "run {run_id} is already completed; \
              create a new run or resume a running/failed one"

From e136973cb63757d0ba5a8260abb25b31e9567dc3 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 11:30:30 -0700
Subject: [PATCH 03/26] adding TODO

---
 cli/src/commands/resume.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs
index 1758ec3..59d1b27 100644
--- a/cli/src/commands/resume.rs
+++ b/cli/src/commands/resume.rs
@@ -38,6 +38,7 @@ pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<(
         println!("Resuming eval run {run_id} ({workflow_name})");
     }
 
+    // TODO: on resume, should we look up the command from the DB, not the config file?
     match bench_config {
         Some(bench) => {
             bench.validate()?;

From a84cd0c81a6236b446696a018a0f00b61c333806 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 11:36:25 -0700
Subject: [PATCH 04/26] progress

---
 cli/src/commands/resume.rs |   8 +--
 cli/src/commands/run.rs    |  14 ++--
 cli/src/config.rs          | 128 ++++++++++++++++++++++++-------------
 3 files changed, 93 insertions(+), 57 deletions(-)

diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs
index 59d1b27..447dde2 100644
--- a/cli/src/commands/resume.rs
+++ b/cli/src/commands/resume.rs
@@ -42,8 +42,8 @@ pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<(
     match bench_config {
         Some(bench) => {
             bench.validate()?;
-            match bench.type_ {
-                qt::config::BenchmarkType::Builtin => {
+            match bench {
+                qt::config::BenchmarkConfig::Builtin(_) => {
                     super::run::execute_builtin(
                         &db,
                         &metrics_store,
@@ -55,8 +55,8 @@ pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<(
                     )
                     .await
                 }
-                qt::config::BenchmarkType::CustomCode => {
-                    let command = bench.command.as_ref().unwrap();
+                qt::config::BenchmarkConfig::CustomCode(c) => {
+                    let command = &c.command;
                     super::run::execute_custom(
                         run_id,
                         workflow_name,
diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs
index 440e90d..7883c31 100644
--- a/cli/src/commands/run.rs
+++ b/cli/src/commands/run.rs
@@ -30,9 +30,9 @@ pub async fn run(
     match bench_config {
         Some(bench) => {
             bench.validate()?;
-            match bench.type_ {
-                qt::config::BenchmarkType::Builtin => {
-                    let (effective_input, _) = assemble_builtin_input(Some(bench), cli_input);
+            match bench {
+                qt::config::BenchmarkConfig::Builtin(b) => {
+                    let (effective_input, _) = assemble_builtin_input(Some(b), cli_input);
                     run_builtin_workflow(
                         workflow_name,
                         effective_input.as_deref(),
@@ -41,9 +41,9 @@ pub async fn run(
                     )
                     .await
                 }
-                qt::config::BenchmarkType::CustomCode => {
+                qt::config::BenchmarkConfig::CustomCode(c) => {
                     let (merged_input, overridden_keys) =
-                        merge_inputs(bench.input.as_ref(), cli_input)?;
+                        merge_inputs(c.input.as_ref(), cli_input)?;
                     let warning = if overridden_keys.is_empty() {
                         None
                     } else {
@@ -52,7 +52,7 @@ pub async fn run(
                             overridden_keys.join(", ")
                         ))
                     };
-                    let command = bench.command.as_ref().unwrap();
+                    let command = &c.command;
 
                     let cwd = std::env::current_dir()?;
                     let root = db::resolve_workspace_root(&cwd, true).await?;
@@ -95,7 +95,7 @@ pub async fn run(
 }
 
 fn assemble_builtin_input(
-    bench: Option<&qt::config::BenchmarkConfig>,
+    bench: Option<&qt::config::BuiltinBenchmarkConfig>,
     cli_input: Option<&str>,
 ) -> (Option<String>, Vec<String>) {
     if let Some(cli_str) = cli_input {
diff --git a/cli/src/config.rs b/cli/src/config.rs
index c429257..8f54b73 100644
--- a/cli/src/config.rs
+++ b/cli/src/config.rs
@@ -1,74 +1,110 @@
 use std::collections::HashMap;
 
 use anyhow::{Context, Result, bail};
-use serde::Deserialize;
+use serde::{Deserialize, Deserializer};
 
 use crate::llm::Sampler;
 
-/// Type of benchmark execution.
-#[derive(Debug, Default, Deserialize, Clone, PartialEq)]
-#[serde(rename_all = "snake_case")]
-pub enum BenchmarkType {
-    /// Native builtin eval implemented in the CLI.
-    #[default]
-    Builtin,
-    /// External custom eval run as a child process.
-    CustomCode,
-}
-
 /// Configuration for a single benchmark.
-#[derive(Debug, Deserialize, Default)]
-pub struct BenchmarkConfig {
-    /// Execution type. Defaults to `builtin`.
-    #[serde(default, rename = "type")]
-    pub type_: BenchmarkType,
-    /// Number of samples (rows) to evaluate. (builtin only)
-    pub samples: Option<usize>,
-    /// Which model sampler to use for this benchmark. (builtin only)
-    pub model: Option<Sampler>,
-    /// Maximum concurrent workers for this benchmark. (builtin only)
-    pub max_workers: Option<usize>,
-    /// Command and arguments to execute. (`custom_code` only)
-    pub command: Option<Vec<String>>,
-    /// Structured input object passed to the eval. (`custom_code` only)
-    pub input: Option<serde_json::Value>,
+///
+/// Exactly one of the two variants is deserialized based on the `type` field:
+/// `builtin` (default when absent) or `custom_code`.
+#[derive(Debug, Clone)]
+pub enum BenchmarkConfig {
+    Builtin(BuiltinBenchmarkConfig),
+    CustomCode(CustomCodeBenchmarkConfig),
 }
 
 impl BenchmarkConfig {
-    /// Validate that the benchmark config fields are consistent with its `type`.
+    /// Validate post-deserialization constraints.
     ///
     /// # Errors
     ///
-    /// Returns an error when a required field is missing or a disallowed field is present.
+    /// Returns an error when a field has an invalid value.
     pub fn validate(&self) -> Result<()> {
-        match self.type_ {
-            BenchmarkType::Builtin => {
-                if self.command.is_some() {
-                    bail!("builtin benchmark config cannot have a `command` field");
-                }
-                if self.input.is_some() {
-                    bail!("builtin benchmark config cannot have an `input` field");
-                }
-            }
-            BenchmarkType::CustomCode => {
-                if self.command.as_ref().is_none_or(Vec::is_empty) {
+        match self {
+            BenchmarkConfig::Builtin(_) => Ok(()),
+            BenchmarkConfig::CustomCode(c) => {
+                if let Some(ref input) = c.input && !input.is_object() {
                     bail!(
-                        "custom_code benchmark config must have a non-empty `command` field"
+                        "custom_code benchmark config `input` must be a JSON object / TOML table"
                     );
                 }
-                if let Some(ref input) = self.input
-                    && !input.is_object()
-                {
+                if c.command.is_empty() {
                     bail!(
-                        "custom_code benchmark config `input` must be a JSON object / TOML table"
+                        "custom_code benchmark config must have a non-empty `command` field"
                     );
                 }
+                Ok(())
             }
         }
-        Ok(())
     }
 }
 
+impl<'de> Deserialize<'de> for BenchmarkConfig {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let value =
+            serde_json::Value::deserialize(deserializer).map_err(serde::de::Error::custom)?;
+
+        let type_str = value.get("type").and_then(|v| v.as_str());
+
+        match type_str {
+            Some("custom_code") => {
+                let config = CustomCodeBenchmarkConfig::deserialize(value).map_err(|e| {
+                    serde::de::Error::custom(format!(
+                        "failed to deserialize custom_code benchmark config: {e}"
+                    ))
+                })?;
+                Ok(BenchmarkConfig::CustomCode(config))
+            }
+            Some("builtin") | None => {
+                let config = BuiltinBenchmarkConfig::deserialize(value).map_err(|e| {
+                    serde::de::Error::custom(format!(
+                        "failed to deserialize builtin benchmark config: {e}"
+                    ))
+                })?;
+                Ok(BenchmarkConfig::Builtin(config))
+            }
+            Some(other) => Err(serde::de::Error::custom(format!(
+                "invalid benchmark type `{other}`; expected `builtin` or `custom_code`",
+            ))),
+        }
+    }
+}
+
+/// Built-in benchmark configuration.
+#[derive(Debug, Deserialize, Clone)]
+#[serde(deny_unknown_fields)]
+pub struct BuiltinBenchmarkConfig {
+    #[serde(default = "default_type_builtin", rename = "type")]
+    pub type_: String,
+    /// Number of samples (rows) to evaluate.
+    pub samples: Option<usize>,
+    /// Which model sampler to use for this benchmark.
+    pub model: Option<Sampler>,
+    /// Maximum concurrent workers for this benchmark.
+    pub max_workers: Option<usize>,
+}
+
+fn default_type_builtin() -> String {
+    "builtin".to_owned()
+}
+
+/// Custom-code benchmark configuration.
+#[derive(Debug, Deserialize, Clone)]
+#[serde(deny_unknown_fields)]
+pub struct CustomCodeBenchmarkConfig {
+    #[serde(rename = "type")]
+    pub type_: String,
+    /// Command and arguments to execute.
+    pub command: Vec<String>,
+    /// Structured input object passed to the eval.
+    pub input: Option<serde_json::Value>,
+}
+
 /// Top-level workspace configuration read from `quantiles.toml` or
 /// `.quantiles.toml` in the current working directory.
 #[derive(Debug, Deserialize, Default)]

From 551383889146373ce0ab7687be5749c9ad7593bd Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 11:37:59 -0700
Subject: [PATCH 05/26] progress

---
 cli/examples/configs/custom_code/my_eval.py | 24 +++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 cli/examples/configs/custom_code/my_eval.py

diff --git a/cli/examples/configs/custom_code/my_eval.py b/cli/examples/configs/custom_code/my_eval.py
new file mode 100644
index 0000000..a51f604
--- /dev/null
+++ b/cli/examples/configs/custom_code/my_eval.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+"""A minimal custom eval script for the Quantiles CLI example."""
+
+import json
+import os
+
+run_id = os.environ.get("QUANTILES_RUN_ID", "unknown")
+workflow = os.environ.get("QUANTILES_WORKFLOW_NAME", "unknown")
+base_url = os.environ.get("QUANTILES_BASE_URL", "unknown")
+input_json = os.environ.get("QUANTILES_INPUT", "{}")
+
+try:
+    parsed = json.loads(input_json)
+except json.JSONDecodeError:
+    parsed = {}
+
+print(
+    {
+        "run_id": run_id,
+        "eval_name": workflow,
+        "base_url": base_url,
+        "parsed_input": parsed,
+    }
+)

From 2f8487dcb927c5ad0bcc7436770a1c0138971e00 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 11:38:25 -0700
Subject: [PATCH 06/26] rm shebang

---
 cli/examples/configs/custom_code/my_eval.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cli/examples/configs/custom_code/my_eval.py b/cli/examples/configs/custom_code/my_eval.py
index a51f604..4baa66d 100644
--- a/cli/examples/configs/custom_code/my_eval.py
+++ b/cli/examples/configs/custom_code/my_eval.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 """A minimal custom eval script for the Quantiles CLI example."""
 
 import json

From 9838d295112ee714b781e56ed84166e693f8b923 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 11:54:53 -0700
Subject: [PATCH 07/26] progress on docs

---
 AGENTS.md         | 16 +++-------------
 cli/README.md     | 18 +++++++++++++-----
 cli/src/config.rs |  8 ++++----
 3 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index c477e38..dd1dddc 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -145,7 +145,7 @@ qt run <evaluation> --json
 qt list --json
 qt show <run_id> --json
 qt compare <baseline_run_id> <candidate_run_id> --json
-qt run <evaluation> --resume <run_id> --json
+qt resume <run_id> --json
 ```
 
 Do not silently change evaluation semantics. Changes to prompts, datasets, scorers, rubrics, sampling parameters, judge configuration, model selection, tool configuration, or step inputs can invalidate comparisons. Call out any such changes in handoff.
@@ -304,20 +304,10 @@ Resume only failed or interrupted runs caused by operational issues such as time
 
 Do not resume a completed run. Start a new run instead.
 
-When resuming a run, use the same workflow name and input JSON. For custom evaluations, also use the same command.
-
-Start a new run instead of resuming when the model, prompt, dataset, rubric, workflow input, or scoring logic intentionally changed.
-
-<!-- AARON: review these -->
-
-```bash
-qt run <run_id> --resume --json
-```
-
-For custom evaluations, preserve the original command as well:
+`qt resume` looks up the stored workflow name and input from the database, then reads the benchmark command and any remaining configuration from `quantiles.toml`. Start a new run instead of resuming when the model, prompt, dataset, rubric, workflow input, or scoring logic intentionally changed.
 
 ```bash
-qt run <run_id> --resume --json -- <command>
+qt resume <run_id> --json
 ```
 
 ### Handle security-related content
diff --git a/cli/README.md b/cli/README.md
index b96a8d7..ed98089 100644
--- a/cli/README.md
+++ b/cli/README.md
@@ -16,9 +16,8 @@ A few commands to see `qt` in action:
 # 1. Initialize a workspace
 qt init
 
-# 2. Run an eval — Quantiles auto-starts a local server, records the run, and tears the server
-# down when the command finishes.
-qt run my-eval -- bun run sdk/typescript/examples/run_demo.ts
+# 2. Run a built-in eval
+qt run pubmedqa
 
 # 3. List and inspect what happened
 qt list
@@ -27,16 +26,25 @@ qt show 1
 
 >See [quantiles.io/documentation/reference/cli](https://quantiles.io/documentation/reference/cli) for a detailed list of `qt` commands.
 
+### Custom evaluations
+
+Custom evaluations are configured in `quantiles.toml`. See [`./examples/configs`](./examples/configs) for examples.
+
+```bash
+# Run a custom evaluation defined in quantiles.toml
+qt run my-eval
+```
+
 ### Comparing runs
 
 After iterating on an eval, you can compare two runs to see exactly what changed:
 
 ```bash
 # Run A — baseline
-qt run my-eval -- bun run sdk/typescript/examples/run_demo.ts
+qt run my-eval
 
 # Run B — your latest iteration
-qt run my-eval -- bun run sdk/typescript/examples/run_demo.ts
+qt run my-eval
 
 # See what changed between them
 qt compare 1 2
diff --git a/cli/src/config.rs b/cli/src/config.rs
index 8f54b73..d8b8575 100644
--- a/cli/src/config.rs
+++ b/cli/src/config.rs
@@ -25,15 +25,15 @@ impl BenchmarkConfig {
         match self {
             BenchmarkConfig::Builtin(_) => Ok(()),
             BenchmarkConfig::CustomCode(c) => {
-                if let Some(ref input) = c.input && !input.is_object() {
+                if let Some(ref input) = c.input
+                    && !input.is_object()
+                {
                     bail!(
                         "custom_code benchmark config `input` must be a JSON object / TOML table"
                     );
                 }
                 if c.command.is_empty() {
-                    bail!(
-                        "custom_code benchmark config must have a non-empty `command` field"
-                    );
+                    bail!("custom_code benchmark config must have a non-empty `command` field");
                 }
                 Ok(())
             }

From 3424bef486bbf88f5a6bffcf6f9d3c8f89913f2e Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 12:08:41 -0700
Subject: [PATCH 08/26] docs progress

---
 README.md     | 40 +++++++++++++++++++++++++++++++++++++++-
 cli/README.md | 42 +++++++++++++++++++++++++++++++++++++++---
 2 files changed, 78 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 0ed6cbe..b109c1b 100644
--- a/README.md
+++ b/README.md
@@ -71,7 +71,32 @@ To learn more detail about what you can do with the CLI, see [quantiles.io/docum
 
 ### Customization
 
-You can customize how the CLI executes benchmarks using a `quantiles.toml` or `.quantiles.toml` configuration file. This file can be used to control benchmark execution behavior as well as customize the models, providers, and other settings used during eval runs. See [`./cli/examples/configs`](./cli/examples/configs) for examples and more details.
+You can customize how the CLI executes benchmarks using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory.
+
+For **built-in benchmarks**, configure settings like `samples`, `model`, and `max_workers`:
+
+```toml
+[benchmarks.pubmedqa]
+samples = 50
+model = "openai:gpt-5.4-nano"
+max_workers = 100
+```
+
+For **custom evaluations**, set `type = "custom_code"` and provide the `command` to run:
+
+```toml
+[benchmarks.my-eval]
+type = "custom_code"
+command = ["python", "my_eval.py"]
+
+[benchmarks.my-eval.input]
+foo = "foo_val"
+bar = "bar_val"
+```
+
+The CLI will execute the command with `QUANTILES_RUN_ID`, `QUANTILES_WORKFLOW_NAME`, `QUANTILES_BASE_URL`, and `QUANTILES_INPUT` environment variables injected. If the run fails, you can resume it later with `qt resume <run_id>`.
+
+See [`./cli/examples/configs`](./cli/examples/configs) for complete examples, including a [custom code benchmark example](./cli/examples/configs/custom_code/quantiles.toml).
 
 ## Local-First and Offline by Default
 
@@ -102,6 +127,19 @@ Quantiles also provides a [benchmark hub](https://quantiles.io/benchmark-hub) fo
 
 A custom evaluation is a [Python](https://quantiles.io/documentation/reference/python-sdk) or [TypeScript](https://quantiles.io/documentation/reference/typescript-sdk) program that is run by the `qt` CLI and uses the [Quantiles API](https://quantiles.io/documentation/reference/rest-api) to execute an eval. Your code owns the evaluation logic like loading data, calling a model or agent, scoring outputs, computing metrics, and returning a summary. Quantiles manages [durable steps, step caching, and step resume](https://quantiles.io/documentation/workflows-and-steps), metrics, inputs, outputs, and comparisons.
 
+Custom evaluations are configured in `quantiles.toml` with `type = "custom_code"`:
+
+```toml
+[benchmarks.my-eval]
+type = "custom_code"
+command = ["python", "my_eval.py"]
+
+[benchmarks.my-eval.input]
+dataset = "my_dataset.jsonl"
+```
+
+Run the evaluation with `qt run my-eval`. If it fails, resume it later with `qt resume <run_id>` — the CLI re-reads the command and stored input automatically.
+
 Use custom evaluations when you need to measure behavior that is specific to your product, workflow, prompt, dataset, rubric, or release process.
 
 Read more about how to build and run custom evaluations at [quantiles.io/documentation/custom-evaluations](https://quantiles.io/documentation/custom-evaluations).
diff --git a/cli/README.md b/cli/README.md
index ed98089..2f33191 100644
--- a/cli/README.md
+++ b/cli/README.md
@@ -28,13 +28,27 @@ qt show 1
 
 ### Custom evaluations
 
-Custom evaluations are configured in `quantiles.toml`. See [`./examples/configs`](./examples/configs) for examples.
+Custom evaluations are configured in `quantiles.toml` with `type = "custom_code"`. The `command` array tells the CLI how to execute your eval, and the optional `input` table is passed to your script as `QUANTILES_INPUT`.
+
+```toml
+[benchmarks.my-eval]
+type = "custom_code"
+command = ["python", "my_eval.py"]
+
+[benchmarks.my-eval.input]
+dataset = "my_dataset.jsonl"
+```
 
 ```bash
-# Run a custom evaluation defined in quantiles.toml
+# Run the custom evaluation
 qt run my-eval
+
+# If it fails, resume with only the run ID
+qt resume <run_id>
 ```
 
+See [`./examples/configs`](./examples/configs) for examples, including a [complete custom code example](./examples/configs/custom_code/quantiles.toml).
+
 ### Comparing runs
 
 After iterating on an eval, you can compare two runs to see exactly what changed:
@@ -92,6 +106,28 @@ The Quantiles CLI, `qt`, keeps execution simple: your code runs locally, while `
 
 ## Customization
 
-You can customize how the CLI executes benchmarks using a `quantiles.toml` or `.quantiles.toml` configuration file. This file can be used to control benchmark execution behavior as well as customize the models, providers, and other settings used during eval runs. See [`./cli/examples/configs`](./cli/examples/configs) for examples and more details.
+You can customize how the CLI executes benchmarks using a `quantiles.toml` or `.quantiles.toml` configuration file.
+
+For **built-in benchmarks**, configure settings like `samples`, `model`, and `max_workers`:
+
+```toml
+[benchmarks.pubmedqa]
+samples = 50
+model = "openai:gpt-5.4-nano"
+max_workers = 100
+```
+
+For **custom evaluations**, set `type = "custom_code"` and provide the `command` to run. The optional `input` table is passed to your script as `QUANTILES_INPUT`.
+
+```toml
+[benchmarks.my-eval]
+type = "custom_code"
+command = ["python", "my_eval.py"]
+
+[benchmarks.my-eval.input]
+foo = "foo_val"
+```
+
+See [`./examples/configs`](./examples/configs) for complete examples.
 
 >Note: Quantiles is designed for high-throughput execution and may issue many requests in parallel. Depending on your provider, model, and account limits, benchmark runs can quickly hit API rate limits or concurrency quotas. Consider reducing concurrency or using models/providers with higher rate limits if you encounter throttling. Example configurations illustrate how to do so.

From 172306bdc6d902a508fd7c9120271919c40fa5e7 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 12:14:02 -0700
Subject: [PATCH 09/26] progress

---
 cli/src/commands/run.rs | 7 ++-----
 cli/src/config.rs       | 9 +--------
 2 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs
index 7883c31..4566100 100644
--- a/cli/src/commands/run.rs
+++ b/cli/src/commands/run.rs
@@ -123,14 +123,11 @@ fn assemble_builtin_input(
 }
 
 fn merge_inputs(
-    config_input: Option<&serde_json::Value>,
+    config_input: Option<&HashMap<String, serde_json::Value>>,
     cli_input: Option<&str>,
 ) -> Result<(Option<String>, Vec<String>)> {
     let mut merged = match config_input {
-        Some(v) if v.is_object() => v.clone(),
-        Some(_) => {
-            bail!("config input is not a JSON object");
-        }
+        Some(v) => serde_json::Value::Object(v.clone().into_iter().collect()),
         None => serde_json::Value::Object(serde_json::Map::default()),
     };
 
diff --git a/cli/src/config.rs b/cli/src/config.rs
index d8b8575..990eba0 100644
--- a/cli/src/config.rs
+++ b/cli/src/config.rs
@@ -25,13 +25,6 @@ impl BenchmarkConfig {
         match self {
             BenchmarkConfig::Builtin(_) => Ok(()),
             BenchmarkConfig::CustomCode(c) => {
-                if let Some(ref input) = c.input
-                    && !input.is_object()
-                {
-                    bail!(
-                        "custom_code benchmark config `input` must be a JSON object / TOML table"
-                    );
-                }
                 if c.command.is_empty() {
                     bail!("custom_code benchmark config must have a non-empty `command` field");
                 }
@@ -102,7 +95,7 @@ pub struct CustomCodeBenchmarkConfig {
     /// Command and arguments to execute.
     pub command: Vec<String>,
     /// Structured input object passed to the eval.
-    pub input: Option<serde_json::Value>,
+    pub input: Option<HashMap<String, serde_json::Value>>,
 }
 
 /// Top-level workspace configuration read from `quantiles.toml` or

From 5b18489cf10316020bdbe0fa1476dc8704d623eb Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 13:46:52 -0700
Subject: [PATCH 10/26] making failure possible in the test script

---
 cli/examples/configs/custom_code/my_eval.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cli/examples/configs/custom_code/my_eval.py b/cli/examples/configs/custom_code/my_eval.py
index 4baa66d..9fd84c8 100644
--- a/cli/examples/configs/custom_code/my_eval.py
+++ b/cli/examples/configs/custom_code/my_eval.py
@@ -2,6 +2,7 @@
 
 import json
 import os
+import sys
 
 run_id = os.environ.get("QUANTILES_RUN_ID", "unknown")
 workflow = os.environ.get("QUANTILES_WORKFLOW_NAME", "unknown")
@@ -21,3 +22,7 @@
         "parsed_input": parsed,
     }
 )
+
+if parsed.get("should_fail"):
+    print("Simulated failure triggered by input.", file=sys.stderr)
+    sys.exit(1)

From 708e5f521d01d71d045689e11b7aee5328f05dbd Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 13:56:56 -0700
Subject: [PATCH 11/26] adding tests for config parsing

---
 cli/src/config.rs | 139 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 139 insertions(+)

diff --git a/cli/src/config.rs b/cli/src/config.rs
index 990eba0..37752ff 100644
--- a/cli/src/config.rs
+++ b/cli/src/config.rs
@@ -149,3 +149,142 @@ pub fn load() -> Result<WorkspaceConfig> {
         toml::from_str(&contents).with_context(|| format!("failed to parse {filename}"))?;
     Ok(config)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn deserialize_builtin_without_type() {
+        let toml = r#"
+            [benchmarks.demo]
+            samples = 10
+        "#;
+        let config: WorkspaceConfig = toml::from_str(toml).unwrap();
+        let bench = config.benchmarks.get("demo").unwrap();
+        assert!(matches!(bench, BenchmarkConfig::Builtin(_)));
+        if let BenchmarkConfig::Builtin(b) = bench {
+            assert_eq!(b.type_, "builtin");
+            assert_eq!(b.samples, Some(10));
+            assert!(b.model.is_none());
+        }
+    }
+
+    #[test]
+    fn deserialize_builtin_with_explicit_type() {
+        let toml = r#"
+            [benchmarks.demo]
+            type = "builtin"
+            samples = 5
+            model = "openai:gpt-4"
+        "#;
+        let config: WorkspaceConfig = toml::from_str(toml).unwrap();
+        let bench = config.benchmarks.get("demo").unwrap();
+        assert!(matches!(bench, BenchmarkConfig::Builtin(_)));
+    }
+
+    #[test]
+    fn deserialize_custom_code() {
+        let toml = r#"
+            [benchmarks.my-eval]
+            type = "custom_code"
+            command = ["python", "eval.py"]
+
+            [benchmarks.my-eval.input]
+            foo = "bar"
+        "#;
+        let config: WorkspaceConfig = toml::from_str(toml).unwrap();
+        let bench = config.benchmarks.get("my-eval").unwrap();
+        assert!(matches!(bench, BenchmarkConfig::CustomCode(_)));
+        if let BenchmarkConfig::CustomCode(c) = bench {
+            assert_eq!(c.command, vec!["python", "eval.py"]);
+            assert!(c.input.is_some());
+            assert_eq!(
+                c.input.as_ref().unwrap().get("foo").unwrap().as_str(),
+                Some("bar")
+            );
+        }
+    }
+
+    #[test]
+    fn deserialize_custom_code_without_input() {
+        let toml = r#"
+            [benchmarks.my-eval]
+            type = "custom_code"
+            command = ["sh", "-c", "echo hello"]
+        "#;
+        let config: WorkspaceConfig = toml::from_str(toml).unwrap();
+        let bench = config.benchmarks.get("my-eval").unwrap();
+        assert!(matches!(bench, BenchmarkConfig::CustomCode(_)));
+        if let BenchmarkConfig::CustomCode(c) = bench {
+            assert!(c.input.is_none());
+        }
+    }
+
+    #[test]
+    fn builtin_rejects_command_field() {
+        let toml = r#"
+            [benchmarks.demo]
+            type = "builtin"
+            command = ["echo", "hello"]
+        "#;
+        let result: Result<WorkspaceConfig, _> = toml::from_str(toml);
+        assert!(result.is_err(), "builtin should reject command field");
+    }
+
+    #[test]
+    fn builtin_rejects_input_field() {
+        let toml = r#"
+            [benchmarks.demo]
+            type = "builtin"
+
+            [benchmarks.demo.input]
+            foo = "bar"
+        "#;
+        let result: Result<WorkspaceConfig, _> = toml::from_str(toml);
+        assert!(result.is_err(), "builtin should reject input field");
+    }
+
+    #[test]
+    fn custom_code_rejects_samples_field() {
+        let toml = r#"
+            [benchmarks.my-eval]
+            type = "custom_code"
+            command = ["echo"]
+            samples = 10
+        "#;
+        let result: Result<WorkspaceConfig, _> = toml::from_str(toml);
+        assert!(result.is_err(), "custom_code should reject samples field");
+    }
+
+    #[test]
+    fn validate_rejects_empty_command() {
+        let bench = BenchmarkConfig::CustomCode(CustomCodeBenchmarkConfig {
+            type_: "custom_code".to_owned(),
+            command: vec![],
+            input: None,
+        });
+        let err = bench.validate().unwrap_err();
+        assert!(err.to_string().contains("non-empty `command`"));
+    }
+
+    #[test]
+    fn validate_accepts_nonempty_command() {
+        let bench = BenchmarkConfig::CustomCode(CustomCodeBenchmarkConfig {
+            type_: "custom_code".to_owned(),
+            command: vec!["python".to_owned(), "eval.py".to_owned()],
+            input: None,
+        });
+        bench.validate().unwrap();
+    }
+
+    #[test]
+    fn invalid_type_errors() {
+        let toml = r#"
+            [benchmarks.demo]
+            type = "unknown"
+        "#;
+        let result: Result<WorkspaceConfig, _> = toml::from_str(toml);
+        assert!(result.is_err());
+    }
+}

From 35260bab12e56e9f40401e353e8d407642d6e424 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 13:59:11 -0700
Subject: [PATCH 12/26] specific command

---
 cli/src/commands/resume.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs
index 447dde2..2477ab4 100644
--- a/cli/src/commands/resume.rs
+++ b/cli/src/commands/resume.rs
@@ -20,7 +20,7 @@ pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<(
 
     let run = db::get_run(&db, run_id).await?;
     if run.status == db::RunStatus::Completed {
-        // TODO: json flag here
+        // TODO: output error in JSON if json == true
         bail!(
             "run {run_id} is already completed; \
              create a new run or resume a running/failed one"

From 3038df6b5f277cece3ebb5bf335fa8ecf6c1f996 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:13:52 -0700
Subject: [PATCH 13/26] todo comment

---
 cli/src/commands/resume.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs
index 2477ab4..4240347 100644
--- a/cli/src/commands/resume.rs
+++ b/cli/src/commands/resume.rs
@@ -38,7 +38,10 @@ pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<(
         println!("Resuming eval run {run_id} ({workflow_name})");
     }
 
-    // TODO: on resume, should we look up the command from the DB, not the config file?
+    // TODO: we always re-read the command from the config file on resume.
+    // This behavior implies that, if the config file is edited between
+    // `qt run` and `qt resume` invocations, the resumed run will use the
+    // updated command, not the original one. We should revisit this decision
     match bench_config {
         Some(bench) => {
             bench.validate()?;

From 0d9d51ab0e38ce4e7ca6f5340670d2d3292bfdb0 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:18:29 -0700
Subject: [PATCH 14/26] fixup

---
 cli/src/config.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/cli/src/config.rs b/cli/src/config.rs
index 37752ff..c6de98c 100644
--- a/cli/src/config.rs
+++ b/cli/src/config.rs
@@ -151,12 +151,13 @@ pub fn load() -> Result<WorkspaceConfig> {
 }
 
 #[cfg(test)]
-mod tests {
+    #[expect(clippy::needless_raw_string_hashes)]
+    mod tests {
     use super::*;
 
     #[test]
     fn deserialize_builtin_without_type() {
-        let toml = r#"
+         let toml = r#"
             [benchmarks.demo]
             samples = 10
         "#;

From 30cee682a9133785a30db57babef79b98e92c9d0 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:21:13 -0700
Subject: [PATCH 15/26] readme clarifications

---
 cli/README.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/cli/README.md b/cli/README.md
index 2f33191..e000fae 100644
--- a/cli/README.md
+++ b/cli/README.md
@@ -17,6 +17,10 @@ A few commands to see `qt` in action:
 qt init
 
 # 2. Run a built-in eval
+# 
+# Note that you can also build and run your own custom evals
+# with the `qt` CLI. See the below "Custom evaluations" section
+# for details.
 qt run pubmedqa
 
 # 3. List and inspect what happened
@@ -28,15 +32,13 @@ qt show 1
 
 ### Custom evaluations
 
-Custom evaluations are configured in `quantiles.toml` with `type = "custom_code"`. The `command` array tells the CLI how to execute your eval, and the optional `input` table is passed to your script as `QUANTILES_INPUT`.
+Custom evaluations are denoted in the configuration file with `type = "custom_code"`. The `command` array tells the CLI how to execute your eval, and the optional `input` table is merged with any values passed in the `qt run --input` flag, then passed to your script as `QUANTILES_INPUT`. An example is below
 
 ```toml
 [benchmarks.my-eval]
 type = "custom_code"
 command = ["python", "my_eval.py"]
-
-[benchmarks.my-eval.input]
-dataset = "my_dataset.jsonl"
+input = {dataset = "my_dataset.jsonl"}
 ```
 
 ```bash

From a0e2bc9a5502c3ae84869a60c87ff93aa5d3b294 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:21:55 -0700
Subject: [PATCH 16/26] progress

---
 cli/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cli/README.md b/cli/README.md
index e000fae..0e7f83a 100644
--- a/cli/README.md
+++ b/cli/README.md
@@ -49,7 +49,7 @@ qt run my-eval
 qt resume <run_id>
 ```
 
-See [`./examples/configs`](./examples/configs) for examples, including a [complete custom code example](./examples/configs/custom_code/quantiles.toml).
+See [`examples/configs/custom_code/quantiles.toml`](./examples/configs/custom_code/quantiles.toml) for a complete working example.
 
 ### Comparing runs
 

From ec4950fb7f8dcf41a8c9a144909ec70ad6b84c4d Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:22:55 -0700
Subject: [PATCH 17/26] more config tests

---
 cli/src/config.rs | 56 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 56 insertions(+)

diff --git a/cli/src/config.rs b/cli/src/config.rs
index c6de98c..e7def98 100644
--- a/cli/src/config.rs
+++ b/cli/src/config.rs
@@ -288,4 +288,60 @@ pub fn load() -> Result<WorkspaceConfig> {
         let result: Result<WorkspaceConfig, _> = toml::from_str(toml);
         assert!(result.is_err());
     }
+
+    #[test]
+    fn custom_code_missing_command_errors() {
+        let toml = r#"
+            [benchmarks.my-eval]
+            type = "custom_code"
+        "#;
+        let result: Result<WorkspaceConfig, _> = toml::from_str(toml);
+        assert!(result.is_err(), "custom_code should require command field");
+    }
+
+    #[test]
+    fn custom_code_nested_input_values() {
+        let toml = r#"
+            [benchmarks.my-eval]
+            type = "custom_code"
+            command = ["python", "eval.py"]
+
+            [benchmarks.my-eval.input]
+            dataset = "foo.jsonl"
+
+            [benchmarks.my-eval.input.nested]
+            a = 1
+            b = true
+        "#;
+        let config: WorkspaceConfig = toml::from_str(toml).unwrap();
+        let bench = config.benchmarks.get("my-eval").unwrap();
+        if let BenchmarkConfig::CustomCode(c) = bench {
+            let input = c.input.as_ref().unwrap();
+            assert_eq!(input.get("dataset").unwrap().as_str(), Some("foo.jsonl"));
+            let nested = input.get("nested").unwrap().as_object().unwrap();
+            assert_eq!(nested.get("a").unwrap().as_i64(), Some(1));
+            assert_eq!(nested.get("b").unwrap().as_bool(), Some(true));
+        } else {
+            panic!("expected custom_code config");
+        }
+    }
+
+    #[test]
+    fn custom_code_empty_input_table() {
+        let toml = r#"
+            [benchmarks.my-eval]
+            type = "custom_code"
+            command = ["echo"]
+
+            [benchmarks.my-eval.input]
+        "#;
+        let config: WorkspaceConfig = toml::from_str(toml).unwrap();
+        let bench = config.benchmarks.get("my-eval").unwrap();
+        if let BenchmarkConfig::CustomCode(c) = bench {
+            assert!(c.input.is_some());
+            assert!(c.input.as_ref().unwrap().is_empty());
+        } else {
+            panic!("expected custom_code config");
+        }
+    }
 }

From fecdfd0455695e5f1cc7d897afd30e58f96cfee3 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:28:13 -0700
Subject: [PATCH 18/26] progress

---
 cli/README.md | 57 ++++++++++++++++++++++++++-------------------------
 1 file changed, 29 insertions(+), 28 deletions(-)

diff --git a/cli/README.md b/cli/README.md
index 0e7f83a..af6cb39 100644
--- a/cli/README.md
+++ b/cli/README.md
@@ -51,6 +51,35 @@ qt resume <run_id>
 
 See [`examples/configs/custom_code/quantiles.toml`](./examples/configs/custom_code/quantiles.toml) for a complete working example.
 
+## Configuration files and customization
+
+You can customize how the CLI executes benchmarks using a `quantiles.toml` or `.quantiles.toml` configuration file.
+
+For **built-in benchmarks**, configure settings like `samples`, `model`, and `max_workers`:
+
+```toml
+[benchmarks.pubmedqa]
+samples = 50
+model = "openai:gpt-5.4-nano"
+max_workers = 100
+```
+
+For **custom evaluations**, set `type = "custom_code"` and provide the `command` to run. The optional `input` table is passed to your script as `QUANTILES_INPUT`.
+
+```toml
+[benchmarks.my-eval]
+type = "custom_code"
+command = ["python", "my_eval.py"]
+
+[benchmarks.my-eval.input]
+foo = "foo_val"
+```
+
+See [`./examples/configs`](./examples/configs) for complete working examples.
+
+>Note: Quantiles is designed for high-throughput execution and may issue many requests in parallel. Depending on your provider, model, and account limits, benchmark runs can quickly hit API rate limits or concurrency quotas. Consider reducing concurrency or using models/providers with higher rate limits if you encounter throttling. Example configurations illustrate how to do so.
+
+
 ### Comparing runs
 
 After iterating on an eval, you can compare two runs to see exactly what changed:
@@ -105,31 +134,3 @@ The Quantiles CLI, `qt`, keeps execution simple: your code runs locally, while `
 - **Client** (your script) owns code execution: the server never runs your logic
   - Note that the CLI itself also has built-in benchmarks, which do not involve your code
 - **CLI** reads the same SQLite database the server writes to
-
-## Customization
-
-You can customize how the CLI executes benchmarks using a `quantiles.toml` or `.quantiles.toml` configuration file.
-
-For **built-in benchmarks**, configure settings like `samples`, `model`, and `max_workers`:
-
-```toml
-[benchmarks.pubmedqa]
-samples = 50
-model = "openai:gpt-5.4-nano"
-max_workers = 100
-```
-
-For **custom evaluations**, set `type = "custom_code"` and provide the `command` to run. The optional `input` table is passed to your script as `QUANTILES_INPUT`.
-
-```toml
-[benchmarks.my-eval]
-type = "custom_code"
-command = ["python", "my_eval.py"]
-
-[benchmarks.my-eval.input]
-foo = "foo_val"
-```
-
-See [`./examples/configs`](./examples/configs) for complete examples.
-
->Note: Quantiles is designed for high-throughput execution and may issue many requests in parallel. Depending on your provider, model, and account limits, benchmark runs can quickly hit API rate limits or concurrency quotas. Consider reducing concurrency or using models/providers with higher rate limits if you encounter throttling. Example configurations illustrate how to do so.

From 8e4f49b5421f030c6ef89312a6069de097959a12 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:33:41 -0700
Subject: [PATCH 19/26] m ore run tests

---
 cli/src/commands/run.rs | 96 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 96 insertions(+)

diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs
index 4566100..748c847 100644
--- a/cli/src/commands/run.rs
+++ b/cli/src/commands/run.rs
@@ -556,3 +556,99 @@ fn start_background_server(
 
     Ok((handle, shutdown_tx))
 }
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use serde_json::json;
+
+    #[test]
+    fn merge_inputs_config_only() {
+        let mut config = HashMap::new();
+        config.insert("foo".to_owned(), json!("bar"));
+        let (result, overridden) = super::merge_inputs(Some(&config), None).unwrap();
+        assert_eq!(result, Some(r#"{"foo":"bar"}"#.to_owned()));
+        assert!(overridden.is_empty());
+    }
+
+    #[test]
+    fn merge_inputs_cli_only() {
+        let (result, overridden) = super::merge_inputs(None, Some(r#"{"x":1}"#)).unwrap();
+        assert_eq!(result, Some(r#"{"x":1}"#.to_owned()));
+        assert!(overridden.is_empty());
+    }
+
+    #[test]
+    fn merge_inputs_both_with_overlap() {
+        let mut config = HashMap::new();
+        config.insert("foo".to_owned(), json!("old"));
+        config.insert("bar".to_owned(), json!("baz"));
+
+        let (result, overridden) =
+            super::merge_inputs(Some(&config), Some(r#"{"foo":"new"}"#)).unwrap();
+        let parsed: serde_json::Value = serde_json::from_str(result.as_ref().unwrap()).unwrap();
+        assert_eq!(parsed["foo"], "new");
+        assert_eq!(parsed["bar"], "baz");
+        assert_eq!(overridden, vec!["foo"]);
+    }
+
+    #[test]
+    fn merge_inputs_cli_invalid_json() {
+        let err = super::merge_inputs(None, Some("not json")).unwrap_err();
+        assert!(err.to_string().contains("failed to parse --input as JSON"));
+    }
+
+    #[test]
+    fn merge_inputs_both_empty() {
+        let (result, overridden) = super::merge_inputs(None, None).unwrap();
+        assert!(result.is_none());
+        assert!(overridden.is_empty());
+    }
+
+    #[test]
+    fn assemble_builtin_input_with_cli_override() {
+        let bench = qt::config::BuiltinBenchmarkConfig {
+            type_: "builtin".to_owned(),
+            samples: Some(10),
+            model: None,
+            max_workers: None,
+        };
+        let (input, _) = super::assemble_builtin_input(Some(&bench), Some(r#"{"model":"x"}"#));
+        assert_eq!(input, Some(r#"{"model":"x"}"#.to_owned()));
+    }
+
+    #[test]
+    fn assemble_builtin_input_from_config() {
+        let bench = qt::config::BuiltinBenchmarkConfig {
+            type_: "builtin".to_owned(),
+            samples: Some(5),
+            model: Some(qt::llm::Sampler::Random {}),
+            max_workers: Some(8),
+        };
+        let (input, _) = super::assemble_builtin_input(Some(&bench), None);
+        let parsed: serde_json::Value = serde_json::from_str(&input.unwrap()).unwrap();
+        assert_eq!(parsed["limit"], 5);
+        assert_eq!(parsed["model"], "random");
+        assert_eq!(parsed["max_workers"], 8);
+    }
+
+    #[test]
+    fn assemble_builtin_input_none_when_no_config_fields() {
+        let bench = qt::config::BuiltinBenchmarkConfig {
+            type_: "builtin".to_owned(),
+            samples: None,
+            model: None,
+            max_workers: None,
+        };
+        let (input, _) = super::assemble_builtin_input(Some(&bench), None);
+        assert!(input.is_none());
+    }
+
+    #[test]
+    fn assemble_builtin_input_none_when_no_bench() {
+        let (input, _) = super::assemble_builtin_input(None, None);
+        assert!(input.is_none());
+    }
+}
+

From cc5c735b2d7bbb394fcc73b22ab0950111a22c33 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:34:33 -0700
Subject: [PATCH 20/26] minor resume refactor + tests

---
 cli/src/commands/resume.rs | 201 +++++++++++++++++++++++++++----------
 1 file changed, 149 insertions(+), 52 deletions(-)

diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs
index 4240347..7d69798 100644
--- a/cli/src/commands/resume.rs
+++ b/cli/src/commands/resume.rs
@@ -4,8 +4,57 @@ use anyhow::{Result, bail};
 
 use qt::builtins;
 use qt::db;
+use qt::db::RunStatus;
 use qt::metrics_store::MetricsStore;
 
+/// Result of planning how to resume a run.
+#[derive(Debug)]
+pub(crate) enum ResumePlan {
+    Builtin,
+    CustomCode(Vec<String>),
+}
+
+/// Plan how to resume a run without doing any IO.
+///
+/// # Errors
+///
+/// Returns an error when the run is already completed, the benchmark config is
+/// invalid, or there is no way to resume the workflow.
+pub(crate) fn plan_resume(
+    workflow_name: &str,
+    run_status: &RunStatus,
+    bench_config: Option<&qt::config::BenchmarkConfig>,
+) -> Result<ResumePlan> {
+    if *run_status == RunStatus::Completed {
+        bail!(
+            "run is already completed; \
+             create a new run or resume a running/failed one"
+        );
+    }
+
+    match bench_config {
+        Some(bench) => {
+            bench.validate()?;
+            match bench {
+                qt::config::BenchmarkConfig::Builtin(_) => Ok(ResumePlan::Builtin),
+                qt::config::BenchmarkConfig::CustomCode(c) => {
+                    Ok(ResumePlan::CustomCode(c.command.clone()))
+                }
+            }
+        }
+        None => {
+            if builtins::resolve(workflow_name).is_some() {
+                Ok(ResumePlan::Builtin)
+            } else {
+                bail!(
+                    "no config section found for benchmark `{workflow_name}`; \
+                     cannot resume custom eval without config"
+                );
+            }
+        }
+    }
+}
+
 /// Resume an existing eval run.
 ///
 /// # Errors
@@ -19,7 +68,7 @@ pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<(
     let metrics_store = MetricsStore::new(db::metrics_dir(&root))?;
 
     let run = db::get_run(&db, run_id).await?;
-    if run.status == db::RunStatus::Completed {
+    if run.status == RunStatus::Completed {
         // TODO: output error in JSON if json == true
         bail!(
             "run {run_id} is already completed; \
@@ -33,64 +82,112 @@ pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<(
     let config = qt::config::load()?;
     let bench_config = config.benchmarks.get(workflow_name);
 
+    let plan = plan_resume(workflow_name, &run.status, bench_config)?;
+
     db::resume_run(&db, run_id).await?;
     if !json {
         println!("Resuming eval run {run_id} ({workflow_name})");
     }
 
     // TODO: we always re-read the command from the config file on resume.
-    // This behavior implies that, if the config file is edited between
-    // `qt run` and `qt resume` invocations, the resumed run will use the
-    // updated command, not the original one. We should revisit this decision
-    match bench_config {
-        Some(bench) => {
-            bench.validate()?;
-            match bench {
-                qt::config::BenchmarkConfig::Builtin(_) => {
-                    super::run::execute_builtin(
-                        &db,
-                        &metrics_store,
-                        run_id,
-                        workflow_name,
-                        stored_input,
-                        json,
-                        process_start,
-                    )
-                    .await
-                }
-                qt::config::BenchmarkConfig::CustomCode(c) => {
-                    let command = &c.command;
-                    super::run::execute_custom(
-                        run_id,
-                        workflow_name,
-                        stored_input,
-                        command,
-                        json,
-                        process_start,
-                        None,
-                    )
-                    .await
-                }
-            }
+    // This means that if the config file is edited between `qt run` and
+    // `qt resume`, the resumed run will use the updated command. It may be
+    // wise to revisit this policy.
+    match plan {
+        ResumePlan::Builtin => {
+            super::run::execute_builtin(
+                &db,
+                &metrics_store,
+                run_id,
+                workflow_name,
+                stored_input,
+                json,
+                process_start,
+            )
+            .await
         }
-        None => {
-            if builtins::resolve(workflow_name).is_some() {
-                super::run::execute_builtin(
-                    &db,
-                    &metrics_store,
-                    run_id,
-                    workflow_name,
-                    stored_input,
-                    json,
-                    process_start,
-                )
-                .await
-            } else {
-                bail!(
-                    "no config section found for benchmark `{workflow_name}`; \
-                     cannot resume custom eval without config"
-                );
-            }
+        ResumePlan::CustomCode(command) => {
+            super::run::execute_custom(
+                run_id,
+                workflow_name,
+                stored_input,
+                &command,
+                json,
+                process_start,
+                None,
+            )
+            .await
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn plan_resume_completed_run_errors() {
+        let bench = qt::config::BenchmarkConfig::Builtin(qt::config::BuiltinBenchmarkConfig {
+            type_: "builtin".to_owned(),
+            samples: None,
+            model: None,
+            max_workers: None,
+        });
+        let err = plan_resume("demo", &RunStatus::Completed, Some(&bench)).unwrap_err();
+        assert!(err.to_string().contains("already completed"));
+    }
+
+    #[test]
+    fn plan_resume_builtin_with_config() {
+        let bench = qt::config::BenchmarkConfig::Builtin(qt::config::BuiltinBenchmarkConfig {
+            type_: "builtin".to_owned(),
+            samples: Some(10),
+            model: None,
+            max_workers: None,
+        });
+        let plan = plan_resume("demo", &RunStatus::Failed, Some(&bench)).unwrap();
+        assert!(matches!(plan, ResumePlan::Builtin));
+    }
+
+    #[test]
+    fn plan_resume_builtin_without_config() {
+        let plan = plan_resume("pubmedqa", &RunStatus::Failed, None).unwrap();
+        assert!(matches!(plan, ResumePlan::Builtin));
+    }
+
+    #[test]
+    fn plan_resume_custom_code_with_config() {
+        let bench =
+            qt::config::BenchmarkConfig::CustomCode(qt::config::CustomCodeBenchmarkConfig {
+                type_: "custom_code".to_owned(),
+                command: vec!["python".to_owned(), "eval.py".to_owned()],
+                input: None,
+            });
+        let plan = plan_resume("my-eval", &RunStatus::Failed, Some(&bench)).unwrap();
+        assert!(matches!(&plan, ResumePlan::CustomCode(cmd) if cmd == &["python", "eval.py"]));
+    }
+
+    #[test]
+    fn plan_resume_custom_code_without_config_errors() {
+        let err = plan_resume("my-eval", &RunStatus::Failed, None).unwrap_err();
+        assert!(err.to_string().contains("no config section found"));
+    }
+
+    #[test]
+    fn plan_resume_unknown_without_config_errors() {
+        let err = plan_resume("unknown-eval", &RunStatus::Failed, None).unwrap_err();
+        assert!(err.to_string().contains("no config section found"));
+    }
+
+    #[test]
+    fn plan_resume_validates_config() {
+        let bench =
+            qt::config::BenchmarkConfig::CustomCode(qt::config::CustomCodeBenchmarkConfig {
+                type_: "custom_code".to_owned(),
+                command: vec![],
+                input: None,
+            });
+        let err = plan_resume("my-eval", &RunStatus::Failed, Some(&bench)).unwrap_err();
+        assert!(err.to_string().contains("non-empty `command`"));
+    }
+}

From 3f0346cc59daacf5074d7a288de3699774d3f701 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:36:03 -0700
Subject: [PATCH 21/26] test rustdoc

---
 cli/src/commands/resume.rs | 14 ++++++++++++++
 cli/src/commands/run.rs    | 18 ++++++++++++++++++
 cli/src/config.rs          |  6 ++++++
 3 files changed, 38 insertions(+)

diff --git a/cli/src/commands/resume.rs b/cli/src/commands/resume.rs
index 7d69798..91b61af 100644
--- a/cli/src/commands/resume.rs
+++ b/cli/src/commands/resume.rs
@@ -125,6 +125,8 @@ pub async fn resume(run_id: i64, json: bool, process_start: Instant) -> Result<(
 mod tests {
     use super::*;
 
+    /// Resuming a run whose status is `completed` must be rejected before any execution
+    /// begins, because a completed run cannot be meaningfully resumed.
     #[test]
     fn plan_resume_completed_run_errors() {
         let bench = qt::config::BenchmarkConfig::Builtin(qt::config::BuiltinBenchmarkConfig {
@@ -137,6 +139,8 @@ mod tests {
         assert!(err.to_string().contains("already completed"));
     }
 
+    /// A builtin benchmark with a valid config section should plan to resume as a builtin,
+    /// using the stored input from the database.
     #[test]
     fn plan_resume_builtin_with_config() {
         let bench = qt::config::BenchmarkConfig::Builtin(qt::config::BuiltinBenchmarkConfig {
@@ -149,12 +153,16 @@ mod tests {
         assert!(matches!(plan, ResumePlan::Builtin));
     }
 
+    /// A builtin benchmark that has no config section can still be resumed by name lookup,
+    /// falling back to the hardcoded builtin registry.
     #[test]
     fn plan_resume_builtin_without_config() {
         let plan = plan_resume("pubmedqa", &RunStatus::Failed, None).unwrap();
         assert!(matches!(plan, ResumePlan::Builtin));
     }
 
+    /// A `custom_code` benchmark with a config section should plan to resume by re-running
+    /// the command array from the config file with the stored DB input.
     #[test]
     fn plan_resume_custom_code_with_config() {
         let bench =
@@ -167,18 +175,24 @@ mod tests {
         assert!(matches!(&plan, ResumePlan::CustomCode(cmd) if cmd == &["python", "eval.py"]));
     }
 
+    /// A `custom_code` benchmark without a config section cannot be resumed because the
+    /// CLI has no source of truth for what command to execute.
     #[test]
     fn plan_resume_custom_code_without_config_errors() {
         let err = plan_resume("my-eval", &RunStatus::Failed, None).unwrap_err();
         assert!(err.to_string().contains("no config section found"));
     }
 
+    /// An unknown workflow name with neither a config section nor a builtin match must
+    /// fail immediately with a clear "no config section found" message.
     #[test]
     fn plan_resume_unknown_without_config_errors() {
         let err = plan_resume("unknown-eval", &RunStatus::Failed, None).unwrap_err();
         assert!(err.to_string().contains("no config section found"));
     }
 
+    /// Even when a config section is present, it must pass `validate()` before resume
+    /// proceeds, so invalid configs (e.g. empty command) are caught early.
     #[test]
     fn plan_resume_validates_config() {
         let bench =
diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs
index 748c847..6c5825a 100644
--- a/cli/src/commands/run.rs
+++ b/cli/src/commands/run.rs
@@ -563,6 +563,8 @@ mod tests {
 
     use serde_json::json;
 
+    /// When only config input is present and no `--input` CLI flag is given, the result
+    /// should be the exact config object with no overridden keys.
     #[test]
     fn merge_inputs_config_only() {
         let mut config = HashMap::new();
@@ -572,6 +574,8 @@ mod tests {
         assert!(overridden.is_empty());
     }
 
+    /// When only a `--input` CLI flag is given with no config input, the result should be
+    /// the parsed CLI JSON object with no overridden keys.
     #[test]
     fn merge_inputs_cli_only() {
         let (result, overridden) = super::merge_inputs(None, Some(r#"{"x":1}"#)).unwrap();
@@ -579,6 +583,8 @@ mod tests {
         assert!(overridden.is_empty());
     }
 
+    /// When both config and CLI inputs specify the same key, the CLI value should win and
+    /// the key should be recorded in the overridden list for the warning.
     #[test]
     fn merge_inputs_both_with_overlap() {
         let mut config = HashMap::new();
@@ -593,12 +599,16 @@ mod tests {
         assert_eq!(overridden, vec!["foo"]);
     }
 
+    /// Passing a non-JSON string to `--input` should produce a clear error so the user
+    /// knows their CLI argument is malformed.
     #[test]
     fn merge_inputs_cli_invalid_json() {
         let err = super::merge_inputs(None, Some("not json")).unwrap_err();
         assert!(err.to_string().contains("failed to parse --input as JSON"));
     }
 
+    /// When neither config nor CLI provides any input keys, the merged result should be
+    /// `None` so the run stores no input JSON.
     #[test]
     fn merge_inputs_both_empty() {
         let (result, overridden) = super::merge_inputs(None, None).unwrap();
@@ -606,6 +616,8 @@ mod tests {
         assert!(overridden.is_empty());
     }
 
+    /// A `--input` CLI flag should take precedence over any config fields, returning the
+    /// raw CLI string directly without assembling a config-based JSON object.
     #[test]
     fn assemble_builtin_input_with_cli_override() {
         let bench = qt::config::BuiltinBenchmarkConfig {
@@ -618,6 +630,8 @@ mod tests {
         assert_eq!(input, Some(r#"{"model":"x"}"#.to_owned()));
     }
 
+    /// When no `--input` is given but the config has builtin fields, they should be
+     assembled into a `BuiltinConfigInput` JSON object with the correct key names.
     #[test]
     fn assemble_builtin_input_from_config() {
         let bench = qt::config::BuiltinBenchmarkConfig {
@@ -633,6 +647,8 @@ mod tests {
         assert_eq!(parsed["max_workers"], 8);
     }
 
+    /// When the builtin config section exists but has no runtime-relevant fields, the
+    /// input should be `None` rather than an empty JSON object.
     #[test]
     fn assemble_builtin_input_none_when_no_config_fields() {
         let bench = qt::config::BuiltinBenchmarkConfig {
@@ -645,6 +661,8 @@ mod tests {
         assert!(input.is_none());
     }
 
+    /// When there is no config section at all and no CLI `--input`, builtin runs should
+    /// proceed with no input JSON stored in the database.
     #[test]
     fn assemble_builtin_input_none_when_no_bench() {
         let (input, _) = super::assemble_builtin_input(None, None);
diff --git a/cli/src/config.rs b/cli/src/config.rs
index e7def98..fd1b240 100644
--- a/cli/src/config.rs
+++ b/cli/src/config.rs
@@ -289,6 +289,8 @@ pub fn load() -> Result<WorkspaceConfig> {
         assert!(result.is_err());
     }
 
+    /// `custom_code` benchmarks must have a `command` field; omitting it should fail at
+    /// parse time because the struct requires it.
     #[test]
     fn custom_code_missing_command_errors() {
         let toml = r#"
@@ -299,6 +301,8 @@ pub fn load() -> Result<WorkspaceConfig> {
         assert!(result.is_err(), "custom_code should require command field");
     }
 
+    /// Nested TOML tables inside `input` should deserialize into nested JSON objects within
+    /// the `HashMap<String, serde_json::Value>`.
     #[test]
     fn custom_code_nested_input_values() {
         let toml = r#"
@@ -326,6 +330,8 @@ pub fn load() -> Result<WorkspaceConfig> {
         }
     }
 
+    /// An empty `[benchmarks.x.input]` TOML section should deserialize as an empty
+    /// `HashMap`, not fail or become `None`.
     #[test]
     fn custom_code_empty_input_table() {
         let toml = r#"

From be34eebe4119bb4039159816bed259da087e08fd Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:36:42 -0700
Subject: [PATCH 22/26] line wrap

---
 cli/src/commands/run.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs
index 6c5825a..a29b3b2 100644
--- a/cli/src/commands/run.rs
+++ b/cli/src/commands/run.rs
@@ -631,7 +631,7 @@ mod tests {
     }
 
     /// When no `--input` is given but the config has builtin fields, they should be
-     assembled into a `BuiltinConfigInput` JSON object with the correct key names.
+    /// assembled into a `BuiltinConfigInput` JSON object with the correct key names.
     #[test]
     fn assemble_builtin_input_from_config() {
         let bench = qt::config::BuiltinBenchmarkConfig {

From 3ab86930ff32eb249b998923e0f1cfb4a34e02e1 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:39:38 -0700
Subject: [PATCH 23/26] spacing

---
 cli/src/config.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cli/src/config.rs b/cli/src/config.rs
index fd1b240..0da4a18 100644
--- a/cli/src/config.rs
+++ b/cli/src/config.rs
@@ -151,13 +151,13 @@ pub fn load() -> Result<WorkspaceConfig> {
 }
 
 #[cfg(test)]
-    #[expect(clippy::needless_raw_string_hashes)]
-    mod tests {
+#[expect(clippy::needless_raw_string_hashes)]
+mod tests {
     use super::*;
 
     #[test]
     fn deserialize_builtin_without_type() {
-         let toml = r#"
+        let toml = r#"
             [benchmarks.demo]
             samples = 10
         "#;

From 64e93b90e755d3ebd31cc6de3bb977b69d339c72 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:40:31 -0700
Subject: [PATCH 24/26] fmt

---
 cli/src/commands/run.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cli/src/commands/run.rs b/cli/src/commands/run.rs
index a29b3b2..18fcb06 100644
--- a/cli/src/commands/run.rs
+++ b/cli/src/commands/run.rs
@@ -669,4 +669,3 @@ mod tests {
         assert!(input.is_none());
     }
 }
-

From 935b7b0459fe7148f9c492d4ddf55ede6b1b3a98 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:42:19 -0700
Subject: [PATCH 25/26] progress

---
 CONFIG.md | 178 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 README.md |   7 ++-
 2 files changed, 183 insertions(+), 2 deletions(-)
 create mode 100644 CONFIG.md

diff --git a/CONFIG.md b/CONFIG.md
new file mode 100644
index 0000000..2fce211
--- /dev/null
+++ b/CONFIG.md
@@ -0,0 +1,178 @@
+# Configuration Guide
+
+Quantiles uses a single TOML configuration file in the current working directory to describe how benchmarks and custom evaluations are executed. The CLI looks for `quantiles.toml` or `.quantiles.toml` in the current working directory. If both exist, the CLI exits with an ambiguity error.
+
+## When you need a config file
+
+You don't need a config file to run built-in benchmarks. `qt run pubmedqa` works out of the box. You do, however, need a config file when you want to do one or more of the following:
+
+- Override built-in benchmark defaults (e.g. model, sample limit)
+- Define custom evaluations (`type = "custom_code"`)
+- Resume custom evaluations later with `qt resume <run_id>`
+
+## File location and name
+
+The CLI looks in this order:
+
+1. `./quantiles.toml`
+2. `./.quantiles.toml`
+
+Only one may exist in a given directory. The search is rooted at the current working directory where you invoke `qt run` or `qt resume`.
+
+## Top-level structure
+
+Every benchmark lives under the `[benchmarks.<name>]` section. The section name is the workflow name you pass to `qt run <name>`.
+
+```toml
+[benchmarks.pubmedqa]
+# builtin fields...
+
+[benchmarks.my-eval]
+type = "custom_code"
+# custom_code fields...
+```
+
+## Benchmark types
+
+Every benchmark section has a `type` field. Valid values are `"builtin"` (default when absent) and `"custom_code"`.
+
+### `builtin`
+
+Built-in benchmarks run natively inside the CLI. Their config sections may contain:
+
+| Field | Type | Required | Description |
+|-------|------|----------|-------------|
+| `type` | string | no | Always `"builtin"`. May be omitted. |
+| `samples` | integer | no | Number of dataset rows to evaluate. |
+| `model` | string or table | no | Model sampler. See [model format](#model-format). |
+| `max_workers` | integer | no | Maximum concurrent workers. |
+
+If none of these fields are present, the built-in uses its own defaults and no config JSON is generated.
+
+### `custom_code`
+
+Custom evaluations are external programs run as child processes. Their config sections **must** contain:
+
+| Field | Type | Required | Description |
+|-------|------|----------|-------------|
+| `type` | string | yes | Must be `"custom_code"`. |
+| `command` | array of strings | yes | Command and arguments to execute. |
+| `input` | table | no | Structured input passed to the child as `QUANTILES_INPUT`. |
+
+The CLI injects these environment variables into the child process:
+
+- `QUANTILES_RUN_ID` — the run ID
+- `QUANTILES_WORKFLOW_NAME` — the benchmark name
+- `QUANTILES_BASE_URL` — local API base URL
+- `QUANTILES_INPUT` — JSON string from the `input` table
+
+## Model format
+
+The `model` field accepts either a provider-prefixed string or a table.
+
+### String form
+
+```toml
+model = "openai:gpt-5.4-nano"
+```
+
+Supported provider prefixes: `openai:`, `anthropic:`, `gemini:`, `cloudflare:`, `random`, `random_label`.
+
+### Table form
+
+```toml
+model = { provider = "cloudflare", model_id = "@cf/..." }
+```
+
+## Input tables
+
+For `custom_code` benchmarks, `input` is an arbitrary TOML table that becomes a JSON object:
+
+```toml
+[benchmarks.my-eval]
+type = "custom_code"
+command = ["python", "eval.py"]
+
+[benchmarks.my-eval.input]
+dataset = "my_data.jsonl"
+max_samples = 100
+
+[benchmarks.my-eval.input.nested]
+foo = "bar"
+```
+
+This produces:
+
+```json
+{"dataset":"my_data.jsonl","max_samples":100,"nested":{"foo":"bar"}}
+```
+
+An empty input table (`[benchmarks.my-eval.input]` with no keys) deserializes as an empty JSON object.
+
+## CLI `--input` overrides
+
+You can override or extend config input at runtime:
+
+```bash
+qt run my-eval --input '{"max_samples":50}'
+```
+
+The CLI merges the `--input` JSON object into the config `input` table. If a key exists in both, the CLI value wins and a warning is printed:
+
+```
+Warning: --input overrides config input for keys: max_samples
+```
+
+In `--json` mode, the warning is included in the JSON output under the `warning` key.
+
+## Config validation
+
+The CLI validates benchmark configs before execution:
+
+- `builtin` sections may **not** contain `command` or `input` fields.
+- `custom_code` sections **must** contain a non-empty `command` array.
+- `custom_code` sections may **not** contain builtin-only fields like `samples` or `model`.
+- Unknown `type` values are rejected.
+
+Validation failures produce clear error messages before any run is created.
+
+## Resuming and the config file
+
+When you run `qt resume <run_id>`, the CLI looks up the stored workflow name and input from the database, then re-reads the command from the current config file. This means:
+
+- You do **not** need to re-pass `--input` on resume.
+- If you edited the config file between `qt run` and `qt resume`, the resumed run uses the **updated** command.
+- If the config section has been removed, resuming a `custom_code` benchmark fails with "no config section found".
+
+## Complete examples
+
+### Built-in with model override
+
+```toml
+[benchmarks.pubmedqa]
+samples = 50
+model = "openai:gpt-5.4-nano"
+max_workers = 100
+```
+
+### Built-in with demo model (no fields needed)
+
+```toml
+[benchmarks.simpleqa-verified]
+samples = 10
+```
+
+### Custom evaluation
+
+```toml
+[benchmarks.hello]
+type = "custom_code"
+command = ["python3", "hello.py"]
+
+[benchmarks.hello.input]
+greeting = "world"
+```
+
+### Custom evaluation with failure simulation
+
+See [`cli/examples/configs/custom_code/quantiles.toml`](./cli/examples/configs/custom_code/quantiles.toml) for a complete, commented example including a sample Python script.
diff --git a/README.md b/README.md
index b109c1b..998c3df 100644
--- a/README.md
+++ b/README.md
@@ -69,7 +69,7 @@ qt compare <run_id_a> <run_id_b>
 
 To learn more detail about what you can do with the CLI, see [quantiles.io/documentation/reference/cli](https://quantiles.io/documentation/reference/cli).
 
-### Customization
+### Configuration file and customization
 
 You can customize how the CLI executes benchmarks using a `quantiles.toml` or `.quantiles.toml` configuration file in the current working directory.
 
@@ -96,7 +96,10 @@ bar = "bar_val"
 
 The CLI will execute the command with `QUANTILES_RUN_ID`, `QUANTILES_WORKFLOW_NAME`, `QUANTILES_BASE_URL`, and `QUANTILES_INPUT` environment variables injected. If the run fails, you can resume it later with `qt resume <run_id>`.
 
-See [`./cli/examples/configs`](./cli/examples/configs) for complete examples, including a [custom code benchmark example](./cli/examples/configs/custom_code/quantiles.toml).
+See the following resources for more details:
+
+- [`CONFIG.md`](./CONFIG.md) - in-depth guides to configuration and reference
+- [`./cli/examples/configs`](./cli/examples/configs) - complete examples, including a [custom code benchmark example](./cli/examples/configs/custom_code/quantiles.toml)
 
 ## Local-First and Offline by Default
 

From 9f0c00ecf346aede8c662f4bdee73e574b35d2a1 Mon Sep 17 00:00:00 2001
From: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
Date: Fri, 19 Jun 2026 20:46:23 -0700
Subject: [PATCH 26/26] more custom config reference

---
 CONFIG.md | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/CONFIG.md b/CONFIG.md
index 2fce211..3155f93 100644
--- a/CONFIG.md
+++ b/CONFIG.md
@@ -12,12 +12,12 @@ You don't need a config file to run built-in benchmarks. `qt run pubmedqa` works
 
 ## File location and name
 
-The CLI looks in this order:
+The CLI looks for a configuration file in the current working directory, in this order:
 
 1. `./quantiles.toml`
 2. `./.quantiles.toml`
 
-Only one may exist in a given directory. The search is rooted at the current working directory where you invoke `qt run` or `qt resume`.
+Only one may exist in a given directory.
 
 ## Top-level structure
 
@@ -42,7 +42,7 @@ Built-in benchmarks run natively inside the CLI. Their config sections may conta
 
 | Field | Type | Required | Description |
 |-------|------|----------|-------------|
-| `type` | string | no | Always `"builtin"`. May be omitted. |
+| `type` | string | no | Defaults to `"builtin"`. May be omitted for built-in benchmarks. |
 | `samples` | integer | no | Number of dataset rows to evaluate. |
 | `model` | string or table | no | Model sampler. See [model format](#model-format). |
 | `max_workers` | integer | no | Maximum concurrent workers. |
@@ -51,7 +51,7 @@ If none of these fields are present, the built-in uses its own defaults and no c
 
 ### `custom_code`
 
-Custom evaluations are external programs run as child processes. Their config sections **must** contain:
+Custom evaluations are external programs run as child processes and generally built with one of the Quantiles SDKs. Their config sections contain the following fields:
 
 | Field | Type | Required | Description |
 |-------|------|----------|-------------|
@@ -76,7 +76,12 @@ The `model` field accepts either a provider-prefixed string or a table.
 model = "openai:gpt-5.4-nano"
 ```
 
-Supported provider prefixes: `openai:`, `anthropic:`, `gemini:`, `cloudflare:`, `random`, `random_label`.
+Supported provider prefixes are listed below:
+
+- `openai:`
+- `anthropic:`
+- `gemini:`
+- `cloudflare:`
 
 ### Table form
 
@@ -86,7 +91,7 @@ model = { provider = "cloudflare", model_id = "@cf/..." }
 
 ## Input tables
 
-For `custom_code` benchmarks, `input` is an arbitrary TOML table that becomes a JSON object:
+For `custom_code` benchmarks, `input` is an arbitrary TOML table that becomes a JSON object in the custom eval (e.g. a `dict` in Python and a `Map` in TypeScript):
 
 ```toml
 [benchmarks.my-eval]
@@ -140,8 +145,8 @@ Validation failures produce clear error messages before any run is created.
 
 When you run `qt resume <run_id>`, the CLI looks up the stored workflow name and input from the database, then re-reads the command from the current config file. This means:
 
-- You do **not** need to re-pass `--input` on resume.
-- If you edited the config file between `qt run` and `qt resume`, the resumed run uses the **updated** command.
+- `qt resume` provides no `--input` flag, and you do not need to re-submit input parameters on resume.
+- If you edited the config file between `qt run` and `qt resume`, the resumed run uses the _updated_ command.
 - If the config section has been removed, resuming a `custom_code` benchmark fails with "no config section found".
 
 ## Complete examples