From ddbbbe29ceb3f5754f863cb6d169301a9bcc2c7f Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Wed, 18 Mar 2026 21:41:58 +0100 Subject: [PATCH 1/5] Skip baseline entries when collecting ML runs --- config/forecasters-co2.yaml | 3 --- config/interpolators-co2.yaml | 2 -- src/evalml/config.py | 14 +++++----- tests/unit/test_config.py | 45 +++++++++++++++++++++++++++++++ workflow/rules/common.smk | 16 +++++++++-- workflow/tools/config.schema.json | 25 +++++++++++------ 6 files changed, 83 insertions(+), 22 deletions(-) diff --git a/config/forecasters-co2.yaml b/config/forecasters-co2.yaml index 059ab97e..7a77c70c 100644 --- a/config/forecasters-co2.yaml +++ b/config/forecasters-co2.yaml @@ -15,10 +15,7 @@ runs: config: resources/inference/configs/sgm-forecaster-global_trimedge.yaml extra_requirements: - git+https://github.com/ecmwf/anemoi-inference.git@0.8.3 - -baselines: - baseline: - baseline_id: COSMO-E label: COSMO-E root: /store_new/mch/msopr/ml/COSMO-E steps: 0/120/6 diff --git a/config/interpolators-co2.yaml b/config/interpolators-co2.yaml index 115dd6bc..767ab431 100644 --- a/config/interpolators-co2.yaml +++ b/config/interpolators-co2.yaml @@ -46,8 +46,6 @@ runs: steps: 0/120/6 extra_requirements: - git+https://github.com/ecmwf/anemoi-inference.git@0.8.3 - -baselines: - baseline: baseline_id: COSMO-E-1h label: COSMO-E diff --git a/src/evalml/config.py b/src/evalml/config.py index 802afabb..18270742 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -151,10 +151,10 @@ class InterpolatorConfig(RunConfig): class BaselineConfig(BaseModel): """Configuration for a single baseline to include in the verification.""" - baseline_id: str = Field( - ..., + baseline_id: str | None = Field( + None, min_length=1, - description="Identifier for the baseline, e.g. 'COSMO-E'.", + description="Optional identifier for the baseline, e.g. 'COSMO-E'. Defaults to label when omitted.", ) label: str = Field( ..., @@ -298,13 +298,13 @@ class ConfigModel(BaseModel): description="Optional label for the experiment that will be used in the experiment directory name. Defaults to the config file name if not provided.", ) dates: Dates | ExplicitDates - runs: List[ForecasterItem | InterpolatorItem] = Field( + runs: List[ForecasterItem | InterpolatorItem | BaselineItem] = Field( ..., - description="Dictionary of runs to execute, with run IDs as keys and configurations as values.", + description="List of experiment participants, including forecaster/interpolator ML runs and baselines.", ) baselines: List[BaselineItem] = Field( - ..., - description="Dictionary of baselines to include in the verification.", + default_factory=list, + description="Deprecated top-level baselines list. Prefer defining baseline entries directly in `runs`.", ) truth: TruthConfig | None stratification: Stratification diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 6b452b02..d5928698 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from evalml.config import ConfigModel @@ -25,3 +27,46 @@ def test_example_interpolators_config(example_interpolators_config): del example_interpolators_config["runs"] with pytest.raises(ValueError, match="Field required"): _ = ConfigModel.model_validate(example_interpolators_config) + + +def test_legacy_top_level_baselines_still_supported(example_forecasters_config): + """Top-level `baselines` remains accepted for backward compatibility.""" + + cfg = {k: v for k, v in example_forecasters_config.items() if k != "runs"} + cfg["runs"] = [ + run for run in example_forecasters_config["runs"] if "forecaster" in run + ] + cfg["baselines"] = [ + run for run in example_forecasters_config["runs"] if "baseline" in run + ] + + _ = ConfigModel.model_validate(cfg) + + +def test_workflow_parsing_excludes_baselines_from_run_configs( + example_forecasters_config, +): + """Baseline entries in `runs` should not be treated as ML run configs.""" + + namespace = { + "Path": Path, + "config": example_forecasters_config, + } + common_rules = Path("workflow/rules/common.smk").read_text() + + exec(common_rules, namespace) + + run_configs = namespace["RUN_CONFIGS"] + baseline_configs = namespace["BASELINE_CONFIGS"] + + assert all( + run_config["model_type"] != "baseline" + for run_config in run_configs.values() + ) + assert baseline_configs == { + "COSMO-E": { + "label": "COSMO-E", + "root": "/store_new/mch/msopr/ml/COSMO-E", + "steps": "0/120/6", + } + } diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 9fe6ff09..a039ba21 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -141,6 +141,8 @@ def collect_all_runs() -> dict: runs: dict[str, dict] = {} for run_entry in config["runs"]: model_type = next(iter(run_entry)) + if model_type == "baseline": + continue run_config = run_entry[model_type] runs |= register_run(model_type, run_config) return runs @@ -159,11 +161,21 @@ def collect_all_candidates(): def collect_all_baselines(): """Collect all baselines defined in the configuration.""" baselines = {} - for baseline_entry in copy.deepcopy(config["baselines"]): + + for run_entry in copy.deepcopy(config["runs"]): + if "baseline" not in run_entry: + continue + baseline_config = run_entry["baseline"] + baseline_id = baseline_config.pop("baseline_id", baseline_config["label"]) + baselines[baseline_id] = baseline_config + + # Backward compatibility with legacy top-level `baselines` block. + for baseline_entry in copy.deepcopy(config.get("baselines", [])): baseline_type = next(iter(baseline_entry)) baseline_config = baseline_entry[baseline_type] - baseline_id = baseline_config.pop("baseline_id") + baseline_id = baseline_config.pop("baseline_id", baseline_config["label"]) baselines[baseline_id] = baseline_config + return baselines diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index cc66d7fb..e82db10d 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -4,10 +4,18 @@ "description": "Configuration for a single baseline to include in the verification.", "properties": { "baseline_id": { - "description": "Identifier for the baseline, e.g. 'COSMO-E'.", - "minLength": 1, - "title": "Baseline Id", - "type": "string" + "anyOf": [ + { + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional identifier for the baseline, e.g. 'COSMO-E'. Defaults to label when omitted.", + "title": "Baseline Id" }, "label": { "description": "Label for the baseline that will be used in experiment results such as reports and figures.", @@ -29,7 +37,6 @@ } }, "required": [ - "baseline_id", "label", "root", "steps" @@ -539,7 +546,7 @@ "title": "Dates" }, "runs": { - "description": "Dictionary of runs to execute, with run IDs as keys and configurations as values.", + "description": "List of experiment participants, including forecaster/interpolator ML runs and baselines.", "items": { "anyOf": [ { @@ -547,6 +554,9 @@ }, { "$ref": "#/$defs/InterpolatorItem" + }, + { + "$ref": "#/$defs/BaselineItem" } ] }, @@ -554,7 +564,7 @@ "type": "array" }, "baselines": { - "description": "Dictionary of baselines to include in the verification.", + "description": "Deprecated top-level baselines list. Prefer defining baseline entries directly in `runs`.", "items": { "$ref": "#/$defs/BaselineItem" }, @@ -585,7 +595,6 @@ "description", "dates", "runs", - "baselines", "truth", "stratification", "locations", From 024664b8db144ef5bce8a45288761387ed905b2d Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Thu, 19 Mar 2026 08:28:14 +0100 Subject: [PATCH 2/5] Update configs and formatting --- config/forecasters-co1e.yaml | 2 -- config/forecasters-co2-disentangled.yaml | 2 -- config/forecasters-co2.yaml | 1 + config/forecasters-ich1-oper.yaml | 4 +--- config/forecasters-ich1.yaml | 7 +++---- config/interpolators-co2.yaml | 1 + tests/unit/test_config.py | 3 +-- 7 files changed, 7 insertions(+), 13 deletions(-) diff --git a/config/forecasters-co1e.yaml b/config/forecasters-co1e.yaml index 83846ec4..3997ffee 100644 --- a/config/forecasters-co1e.yaml +++ b/config/forecasters-co1e.yaml @@ -20,9 +20,7 @@ runs: extra_requirements: - git+https://github.com/ecmwf/anemoi-inference.git@0.6.3 -baselines: - baseline: - baseline_id: COSMO-1E label: COSMO-1E root: /store_new/mch/msopr/ml/COSMO-1E steps: 0/33/6 diff --git a/config/forecasters-co2-disentangled.yaml b/config/forecasters-co2-disentangled.yaml index c9a75f2c..63178aec 100644 --- a/config/forecasters-co2-disentangled.yaml +++ b/config/forecasters-co2-disentangled.yaml @@ -41,9 +41,7 @@ runs: - earthkit-data<0.19.0 - git+https://github.com/MeteoSwiss/anemoi-core.git@2a90165e3f25defc55fbeb77f7b4ebfef685820d#subdirectory=models -baselines: - baseline: - baseline_id: COSMO-E label: COSMO-E root: /store_new/mch/msopr/ml/COSMO-E steps: 0/120/6 diff --git a/config/forecasters-co2.yaml b/config/forecasters-co2.yaml index 7a77c70c..e9aeb8db 100644 --- a/config/forecasters-co2.yaml +++ b/config/forecasters-co2.yaml @@ -15,6 +15,7 @@ runs: config: resources/inference/configs/sgm-forecaster-global_trimedge.yaml extra_requirements: - git+https://github.com/ecmwf/anemoi-inference.git@0.8.3 + - baseline: label: COSMO-E root: /store_new/mch/msopr/ml/COSMO-E diff --git a/config/forecasters-ich1-oper.yaml b/config/forecasters-ich1-oper.yaml index 6e5b011f..b8db4cce 100644 --- a/config/forecasters-ich1-oper.yaml +++ b/config/forecasters-ich1-oper.yaml @@ -22,14 +22,12 @@ runs: - earthkit-data<0.19.0 - git+https://github.com/ecmwf/anemoi-inference.git@main -baselines: - baseline: - baseline_id: ICON-CH1-EPS label: ICON-CH1-ctrl root: /scratch/mch/cmerker/ICON-CH1-EPS steps: 0/33/6 + - baseline: - baseline_id: ICON-CH2-EPS label: ICON-CH2-ctrl root: /scratch/mch/cmerker/ICON-CH2-EPS steps: 0/120/6 diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index a5de9b54..f2982946 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -35,11 +35,10 @@ runs: # config: resources/inference/configs/sgm-forecaster-global-ich1.yaml # disable_local_eccodes_definitions: true # extra_requirements: - # - earthkit-utils<0.2.0 - # - earthkit-data<0.19.0 - # - git+https://github.com/ecmwf/anemoi-inference.git@main + # - earthkit-utils<0.2.0 + # - earthkit-data<0.19.0 + # - git+https://github.com/ecmwf/anemoi-inference.git@main -baselines: - baseline: baseline_id: ICON-CH2-EPS label: ICON-CH2-EPS diff --git a/config/interpolators-co2.yaml b/config/interpolators-co2.yaml index 767ab431..21aa8625 100644 --- a/config/interpolators-co2.yaml +++ b/config/interpolators-co2.yaml @@ -46,6 +46,7 @@ runs: steps: 0/120/6 extra_requirements: - git+https://github.com/ecmwf/anemoi-inference.git@0.8.3 + - baseline: baseline_id: COSMO-E-1h label: COSMO-E diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index d5928698..7b1adf92 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -60,8 +60,7 @@ def test_workflow_parsing_excludes_baselines_from_run_configs( baseline_configs = namespace["BASELINE_CONFIGS"] assert all( - run_config["model_type"] != "baseline" - for run_config in run_configs.values() + run_config["model_type"] != "baseline" for run_config in run_configs.values() ) assert baseline_configs == { "COSMO-E": { From 43fd76deca720ebe7b2585a7b85e32493fc00dfa Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Thu, 19 Mar 2026 08:35:27 +0100 Subject: [PATCH 3/5] Fix default baseline id --- workflow/rules/common.smk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index a039ba21..4257b1c6 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -166,7 +166,7 @@ def collect_all_baselines(): if "baseline" not in run_entry: continue baseline_config = run_entry["baseline"] - baseline_id = baseline_config.pop("baseline_id", baseline_config["label"]) + baseline_id = baseline_config["baseline_id"] or baseline_config["label"] baselines[baseline_id] = baseline_config # Backward compatibility with legacy top-level `baselines` block. @@ -223,3 +223,4 @@ REFTIMES = parse_reference_times() RUN_CONFIGS = collect_all_runs() BASELINE_CONFIGS = collect_all_baselines() EXPERIMENT_PARTICIPANTS = collect_experiment_participants() +print(EXPERIMENT_PARTICIPANTS) From 33505f73b37aac9f25358cfe5d80de2c9575c3e3 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Thu, 19 Mar 2026 10:37:37 +0100 Subject: [PATCH 4/5] Derive baseline_id from the root stem --- README.md | 1 - config/forecasters-ich1-oper-fixed.yaml | 1 - config/forecasters-ich1.yaml | 1 - config/interpolators-co2.yaml | 1 - src/evalml/config.py | 4 ++-- tests/unit/test_config.py | 22 ++++++++++++++++++++++ workflow/rules/common.smk | 5 +++-- workflow/tools/config.schema.json | 4 ++-- 8 files changed, 29 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 32ad0a86..1f375503 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,6 @@ runs: baselines: - baseline: - baseline_id: COSMO-E label: COSMO-E root: /store_new/mch/msopr/ml/COSMO-E steps: 0/120/6 diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index 7eb5709b..bd4a6f5e 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -33,7 +33,6 @@ runs: baselines: - baseline: - baseline_id: ICON-CH1-EPS label: ICON-CH1-EPS root: /store_new/mch/msopr/ml/ICON-CH1-EPS steps: 0/33/6 diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index f2982946..752e64bb 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -40,7 +40,6 @@ runs: # - git+https://github.com/ecmwf/anemoi-inference.git@main - baseline: - baseline_id: ICON-CH2-EPS label: ICON-CH2-EPS root: /scratch/mch/cmerker/ICON-CH2-EPS steps: 0/120/6 diff --git a/config/interpolators-co2.yaml b/config/interpolators-co2.yaml index 21aa8625..cbece184 100644 --- a/config/interpolators-co2.yaml +++ b/config/interpolators-co2.yaml @@ -48,7 +48,6 @@ runs: - git+https://github.com/ecmwf/anemoi-inference.git@0.8.3 - baseline: - baseline_id: COSMO-E-1h label: COSMO-E root: /store_new/mch/msopr/ml/COSMO-E_hourly steps: 0/120/1 diff --git a/src/evalml/config.py b/src/evalml/config.py index 18270742..b1b7eab8 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -154,7 +154,7 @@ class BaselineConfig(BaseModel): baseline_id: str | None = Field( None, min_length=1, - description="Optional identifier for the baseline, e.g. 'COSMO-E'. Defaults to label when omitted.", + description="Deprecated compatibility field. Workflow baseline IDs are derived from the stem of `root`.", ) label: str = Field( ..., @@ -164,7 +164,7 @@ class BaselineConfig(BaseModel): root: str = Field( ..., min_length=1, - description="Root directory where the baseline data is stored.", + description="Root directory where the baseline data is stored. The workflow derives the baseline ID from the stem of this path.", ) steps: str = Field( ..., diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 7b1adf92..40281a6e 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -69,3 +69,25 @@ def test_workflow_parsing_excludes_baselines_from_run_configs( "steps": "0/120/6", } } + + +def test_workflow_derives_baseline_id_from_root_stem(example_interpolators_config): + """Workflow baseline IDs should come from the baseline root path stem.""" + + namespace = { + "Path": Path, + "config": example_interpolators_config, + } + common_rules = Path("workflow/rules/common.smk").read_text() + + exec(common_rules, namespace) + + baseline_configs = namespace["BASELINE_CONFIGS"] + + assert "COSMO-E_hourly" in baseline_configs + assert "COSMO-E-1h" not in baseline_configs + assert baseline_configs["COSMO-E_hourly"] == { + "label": "COSMO-E", + "root": "/store_new/mch/msopr/ml/COSMO-E_hourly", + "steps": "0/120/1", + } diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 4257b1c6..0cbef1e2 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -166,14 +166,15 @@ def collect_all_baselines(): if "baseline" not in run_entry: continue baseline_config = run_entry["baseline"] - baseline_id = baseline_config["baseline_id"] or baseline_config["label"] + baseline_id = Path(baseline_config["root"]).stem baselines[baseline_id] = baseline_config # Backward compatibility with legacy top-level `baselines` block. for baseline_entry in copy.deepcopy(config.get("baselines", [])): baseline_type = next(iter(baseline_entry)) baseline_config = baseline_entry[baseline_type] - baseline_id = baseline_config.pop("baseline_id", baseline_config["label"]) + baseline_id = Path(baseline_config["root"]).stem + baseline_config.pop("baseline_id", None) baselines[baseline_id] = baseline_config return baselines diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index e82db10d..20f65126 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -14,7 +14,7 @@ } ], "default": null, - "description": "Optional identifier for the baseline, e.g. 'COSMO-E'. Defaults to label when omitted.", + "description": "Deprecated compatibility field. Workflow baseline IDs are derived from the stem of `root`.", "title": "Baseline Id" }, "label": { @@ -24,7 +24,7 @@ "type": "string" }, "root": { - "description": "Root directory where the baseline data is stored.", + "description": "Root directory where the baseline data is stored. The workflow derives the baseline ID from the stem of this path.", "minLength": 1, "title": "Root", "type": "string" From 6e7a4626bc26058fc961a03b354e6954f913bc73 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Thu, 19 Mar 2026 10:45:47 +0100 Subject: [PATCH 5/5] Aesthetics --- config/forecasters-ich1.yaml | 6 +++--- workflow/rules/common.smk | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index 752e64bb..3e8a9658 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -35,9 +35,9 @@ runs: # config: resources/inference/configs/sgm-forecaster-global-ich1.yaml # disable_local_eccodes_definitions: true # extra_requirements: - # - earthkit-utils<0.2.0 - # - earthkit-data<0.19.0 - # - git+https://github.com/ecmwf/anemoi-inference.git@main + # - earthkit-utils<0.2.0 + # - earthkit-data<0.19.0 + # - git+https://github.com/ecmwf/anemoi-inference.git@main - baseline: label: ICON-CH2-EPS diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 0cbef1e2..3b5062e5 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -224,4 +224,3 @@ REFTIMES = parse_reference_times() RUN_CONFIGS = collect_all_runs() BASELINE_CONFIGS = collect_all_baselines() EXPERIMENT_PARTICIPANTS = collect_experiment_participants() -print(EXPERIMENT_PARTICIPANTS)