Adding hugging-face dataset prefixes

CONFIG.md

@@ -26,6 +26,7 @@ For example, if you want to override default parameters for the built-in PubMedQ
 # Configure the model and sample limit for the built-in PubMedQA benchmark.
 
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
 samples = 50
 model = "openai:gpt-5.6"
 ```
@@ -41,17 +42,32 @@ Built-in benchmarks run natively inside the CLI, without any custom code. Below
 | Field         | Type            | Required | Description                                                      |
 | ------------- | --------------- | -------- | ---------------------------------------------------------------- |
 | `type`        | string          | no       | Defaults to `"builtin"`. May be omitted for built-in benchmarks. |
+| `dataset`     | string          | no       | Dataset source. Hugging Face datasets must use `hf://...`.       |
 | `samples`     | integer         | no       | Number of dataset rows to evaluate.                              |
 | `model`       | string or table | no       | Model sampler. See [model format](#model-format).                |
 | `max_workers` | integer         | no       | Maximum concurrent workers.                                      |
 
 If none of these fields are customized, the built-in benchmark uses the following defaults:
 
 - `type`: `builtin`
+- `dataset`: The benchmark's default Hugging Face dataset.
 - `samples`: All samples available in the benchmark's dataset, in order
 - `model`: The "demo" model, which outputs random values
 - `max_workers`: The default parallelism provided by the Rust [Tokio runtime](https://tokio.rs/)
 
+#### `dataset` naming
+
+The `dataset` field accepts a Hugging Face dataset source prefixed with `hf://`.
+For example:
+
+```toml
+dataset = "hf://quantiles/PubMedQA"
+```
+
+The CLI resolves `hf://quantiles/PubMedQA` to the Hugging Face dataset ID
+`quantiles/PubMedQA` before using the normal Hugging Face download path.
+Other URI-style prefixes are rejected.
+
 #### `model` naming
 
 The `model` field described above accepts a provider-prefixed string, for example:
@@ -156,13 +172,15 @@ When you run `qt resume <run_id>`, the CLI looks up the stored eval name and inp
 
 ```toml
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
 model = "openai:gpt-5.6"
 ```
 
 ### Built-in using the demo model with a sample limit
 
 ```toml
 [benchmarks.simpleqa-verified]
+dataset = "hf://quantiles/simpleqa-verified"
 samples = 10
 ```
 

README.md

@@ -79,6 +79,7 @@ For **built-in benchmarks**, configure settings like `samples`, `model`, and `ma
 
 ```toml
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
 samples = 50
 model = "openai:gpt-5.6"
 max_workers = 100

cli/README.md

@@ -60,10 +60,11 @@ You can customize how the CLI executes built-in benchmarks and custom evaluation
 
 ### Built-in benchmarks
 
-For built-in benchmarks, configure settings like `samples`, `model`, and `max_workers`:
+For built-in benchmarks, configure settings like `dataset`, `samples`, `model`, and `max_workers`:
 
 ```toml
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
 samples = 50
 model = "openai:gpt-5.6"
 max_workers = 100

cli/examples/configs/anthropic/quantiles.toml

@@ -5,6 +5,8 @@
 # variable. To get an API key, go to https://platform.claude.com/settings/workspaces/default/keys
 
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
+
 # The full expert-labeled PubMedQA benchmark has 1000 samples. Since Anthropic charges for
 # usage, we are limiting the number of samples to 50 in this demo, to keep costs lower.
 samples = 50

cli/examples/configs/both_error/.quantiles.toml

@@ -1,8 +1,11 @@
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
 samples = 10
 
 [benchmarks.simpleqa-verified]
+dataset = "hf://quantiles/simpleqa-verified"
 samples = 100
 
 [benchmarks.financebench]
+dataset = "hf://quantiles/financebench"
 samples = 50

cli/examples/configs/cloudflare/quantiles.toml

@@ -13,6 +13,8 @@
 # https://dash.cloudflare.com/<account_id>/ai/ai-gateway
 
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
+
 # The full expert-labeled PubMedQA benchmark has 1000 samples. Since Cloudflare charges for
 # usage, we are limiting the number of samples to 50 in this demo, to keep costs lower.
 samples = 50

cli/examples/configs/gemini/quantiles.toml

@@ -7,6 +7,8 @@
 # https://aistudio.google.com/prompts/new_chat
 
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
+
 # The full expert-labeled PubMedQA benchmark has 1000 samples. Since Google charges for
 # usage, we are limiting the number of samples to 50 in this demo, to keep costs lower.
 samples = 50

cli/examples/configs/openai/quantiles.toml

@@ -8,6 +8,8 @@
 # https://platform.openai.com/home
 
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
+
 # The full expert-labeled PubMedQA benchmark has 1000 samples. Since OpenAI charges for
 # usage, we are limiting the number of samples to 50 in this demo, to keep costs lower.
 samples = 50

cli/examples/configs/simple/quantiles.toml

@@ -8,7 +8,9 @@
 # number of dataset rows processed.
 
 [benchmarks.pubmedqa]
+dataset = "hf://quantiles/PubMedQA"
 samples = 10
 
 [benchmarks.simpleqa-verified]
+dataset = "hf://quantiles/simpleqa-verified"
 samples = 100

cli/src/builtins/common.rs

@@ -18,6 +18,9 @@ pub(crate) struct BuiltinConfig {
     /// Number of dataset rows to evaluate. If omitted, the entire dataset is used.
     #[serde(default)]
     pub(crate) limit: Option<usize>,
+    /// Dataset source to evaluate. Builtins currently support Hugging Face via `hf://...`.
+    #[serde(default)]
+    pub(crate) dataset: Option<String>,
     /// Which model sampler to use. If omitted, the builtin chooses a sensible default.
     #[serde(default)]
     pub(crate) model: Option<Sampler>,

cli/src/builtins/input.rs

@@ -5,6 +5,7 @@ use crate::llm::Sampler;
 /// Normalized run input schema for all builtins.
 #[derive(Serialize)]
 pub(crate) struct BuiltinRunInput {
+    pub(crate) dataset: String,
     pub(crate) model: String,
     pub(crate) num_samples: usize,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -15,11 +16,13 @@ pub(crate) struct BuiltinRunInput {
 pub(crate) async fn set_builtin_run_input(
     db: &sea_orm::DatabaseConnection,
     run_id: i64,
+    dataset: &str,
     model: Option<&Sampler>,
     num_samples: usize,
     max_workers: Option<usize>,
 ) -> anyhow::Result<()> {
     let input = serde_json::to_string(&BuiltinRunInput {
+        dataset: dataset.to_owned(),
         model: builtin_model_name(model),
         num_samples,
         max_workers,

cli/src/builtins/pubmedqa/mod.rs

@@ -7,7 +7,7 @@ use crate::builtins::dataset_runner::DatasetRunner;
 use crate::builtins::input::set_builtin_run_input;
 use crate::builtins::output::set_builtin_run_output;
 use crate::builtins::{BuiltinContext, BuiltinWorkflow};
-use crate::dataset::DatasetManager;
+use crate::dataset::{DatasetManager, resolve_hf_dataset_source};
 use crate::llm::LLMSampler;
 use crate::llm::random_label::RandomLabelSampler;
 
@@ -22,6 +22,8 @@ mod eval;
 /// `PubMedQA` builtin using the quantiles/PubMedQA dataset.
 pub struct PubmedqaBuiltin;
 
+const DEFAULT_DATASET_SOURCE: &str = "hf://quantiles/PubMedQA";
+
 #[expect(clippy::too_many_lines)]
 #[async_trait::async_trait]
 impl BuiltinWorkflow for PubmedqaBuiltin {
@@ -49,7 +51,12 @@ impl BuiltinWorkflow for PubmedqaBuiltin {
         };
 
         let manager = DatasetManager::new()?;
-        let dataset_id = "quantiles/PubMedQA";
+        let dataset_source = config
+            .base
+            .dataset
+            .as_deref()
+            .unwrap_or(DEFAULT_DATASET_SOURCE);
+        let dataset_id = resolve_hf_dataset_source(dataset_source)?;
         let info = manager
             .init(dataset_id, Some("pqa_labeled"), Some("train"), None)
             .await?;
@@ -62,6 +69,7 @@ impl BuiltinWorkflow for PubmedqaBuiltin {
         set_builtin_run_input(
             ctx.db,
             ctx.run_id,
+            dataset_source,
             config.base.model.as_ref(),
             limit,
             config.base.max_workers,

cli/src/builtins/similarity.rs

@@ -9,7 +9,7 @@ use crate::builtins::dataset_runner::DatasetRunner;
 use crate::builtins::input::set_builtin_run_input;
 use crate::builtins::output::set_builtin_run_output;
 use crate::builtins::{BuiltinContext, BuiltinWorkflow};
-use crate::dataset::DatasetManager;
+use crate::dataset::{DatasetManager, resolve_hf_dataset_source};
 use crate::llm::LLMSampler;
 use crate::llm::random::RandomSampler;
 use crate::similarity::{
@@ -45,23 +45,23 @@ struct RowOutput {
 #[derive(Clone, Copy)]
 pub struct SimilarityBenchmark {
     name: &'static str,
-    dataset_id: &'static str,
+    dataset_source: &'static str,
     input_field: &'static str,
     target_field: &'static str,
 }
 
 /// `simpleqa-verified` builtin.
 pub const SIMPLEQA: SimilarityBenchmark = SimilarityBenchmark {
     name: "simpleqa-verified",
-    dataset_id: "quantiles/simpleqa-verified",
+    dataset_source: "hf://quantiles/simpleqa-verified",
     input_field: "problem",
     target_field: "answer",
 };
 
 /// `financebench` builtin.
 pub const FINANCEBENCH: SimilarityBenchmark = SimilarityBenchmark {
     name: "financebench",
-    dataset_id: "quantiles/financebench",
+    dataset_source: "hf://quantiles/financebench",
     input_field: "question",
     target_field: "answer",
 };
@@ -96,7 +96,13 @@ impl BuiltinWorkflow for SimilarityBenchmark {
         };
 
         let manager = DatasetManager::new()?;
-        let info = manager.init(self.dataset_id, None, None, None).await?;
+        let dataset_source = config
+            .base
+            .dataset
+            .as_deref()
+            .unwrap_or(self.dataset_source);
+        let dataset_id = resolve_hf_dataset_source(dataset_source)?;
+        let info = manager.init(dataset_id, None, None, None).await?;
 
         let total = info
             .total_rows
@@ -109,6 +115,7 @@ impl BuiltinWorkflow for SimilarityBenchmark {
         set_builtin_run_input(
             db,
             run_id,
+            dataset_source,
             config.base.model.as_ref(),
             limit,
             config.base.max_workers,
@@ -120,7 +127,7 @@ impl BuiltinWorkflow for SimilarityBenchmark {
         let metric = &metric;
         let max_workers = config.base.max_workers.unwrap_or_else(get_max_workers);
 
-        let scores = DatasetRunner::new(&manager, self.dataset_id, &info, limit)
+        let scores = DatasetRunner::new(&manager, dataset_id, &info, limit)
             .desc(self.name)
             .set_quiet(ctx.quiet)
             .for_each_concurrent(max_workers, |i, row| {

cli/src/commands/resume.rs

@@ -132,6 +132,7 @@ mod tests {
         let bench = qt::config::BenchmarkConfig::Builtin(qt::config::BuiltinBenchmarkConfig {
             type_: "builtin".to_owned(),
             samples: None,
+            dataset: None,
             model: None,
             max_workers: None,
         });
@@ -146,6 +147,7 @@ mod tests {
         let bench = qt::config::BenchmarkConfig::Builtin(qt::config::BuiltinBenchmarkConfig {
             type_: "builtin".to_owned(),
             samples: Some(10),
+            dataset: None,
             model: None,
             max_workers: None,
         });

cli/src/commands/run.rs

@@ -103,12 +103,17 @@ fn assemble_builtin_input(
     }
 
     if let Some(bench) = bench {
-        if bench.samples.is_none() && bench.model.is_none() && bench.max_workers.is_none() {
+        if bench.samples.is_none()
+            && bench.dataset.is_none()
+            && bench.model.is_none()
+            && bench.max_workers.is_none()
+        {
             return (None, Vec::new());
         }
 
         let input = BuiltinConfigInput {
             limit: bench.samples,
+            dataset: bench.dataset.clone(),
             model: bench.model.clone(),
             max_workers: bench.max_workers,
         };
@@ -383,6 +388,8 @@ struct BuiltinConfigInput {
     #[serde(skip_serializing_if = "Option::is_none")]
     limit: Option<usize>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    dataset: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     model: Option<qt::llm::Sampler>,
     #[serde(skip_serializing_if = "Option::is_none")]
     max_workers: Option<usize>,
@@ -623,6 +630,7 @@ mod tests {
         let bench = qt::config::BuiltinBenchmarkConfig {
             type_: "builtin".to_owned(),
             samples: Some(10),
+            dataset: None,
             model: None,
             max_workers: None,
         };
@@ -637,12 +645,14 @@ mod tests {
         let bench = qt::config::BuiltinBenchmarkConfig {
             type_: "builtin".to_owned(),
             samples: Some(5),
+            dataset: Some("hf://quantiles/PubMedQA".to_owned()),
             model: Some(qt::llm::Sampler::Random {}),
             max_workers: Some(8),
         };
         let (input, _) = super::assemble_builtin_input(Some(&bench), None);
         let parsed: serde_json::Value = serde_json::from_str(&input.unwrap()).unwrap();
         assert_eq!(parsed["limit"], 5);
+        assert_eq!(parsed["dataset"], "hf://quantiles/PubMedQA");
         assert_eq!(parsed["model"], "random");
         assert_eq!(parsed["max_workers"], 8);
     }
@@ -654,6 +664,7 @@ mod tests {
         let bench = qt::config::BuiltinBenchmarkConfig {
             type_: "builtin".to_owned(),
             samples: None,
+            dataset: None,
             model: None,
             max_workers: None,
         };

cli/src/config.rs

@@ -76,6 +76,8 @@ pub struct BuiltinBenchmarkConfig {
     pub type_: String,
     /// Number of samples (rows) to evaluate.
     pub samples: Option<usize>,
+    /// Dataset source for this benchmark.
+    pub dataset: Option<String>,
     /// Which model sampler to use for this benchmark.
     pub model: Option<Sampler>,
     /// Maximum concurrent workers for this benchmark.
@@ -167,6 +169,7 @@ mod tests {
         if let BenchmarkConfig::Builtin(b) = bench {
             assert_eq!(b.type_, "builtin");
             assert_eq!(b.samples, Some(10));
+            assert!(b.dataset.is_none());
             assert!(b.model.is_none());
         }
     }
@@ -177,11 +180,15 @@ mod tests {
             [benchmarks.demo]
             type = "builtin"
             samples = 5
+            dataset = "hf://quantiles/demo"
             model = "openai:gpt-4"
         "#;
         let config: WorkspaceConfig = toml::from_str(toml).unwrap();
         let bench = config.benchmarks.get("demo").unwrap();
         assert!(matches!(bench, BenchmarkConfig::Builtin(_)));
+        if let BenchmarkConfig::Builtin(b) = bench {
+            assert_eq!(b.dataset.as_deref(), Some("hf://quantiles/demo"));
+        }
     }
 
     #[test]