diff --git a/config/forecasters-co2-mod-01.yaml b/config/forecasters-co2-mod-01.yaml new file mode 100644 index 00000000..b95c6bad --- /dev/null +++ b/config/forecasters-co2-mod-01.yaml @@ -0,0 +1,56 @@ +# yaml-language-server: $schema=../workflow/tools/config.schema.json +description: | + Evaluate skill of COSMO-E emulator (M-1 forecaster). + +dates: + start: 2020-01-02T00:00 + end: 2020-01-31T00:00 + frequency: 24h + +runs: + - forecaster: + mlflow_id: d0846032fc7248a58b089cbe8fa4c511 + label: M-1 forecaster + steps: 0/120/6 + config: resources/inference/configs/sgm-forecaster-global_trimedge.yaml + extra_dependencies: + - git+https://github.com/ecmwf/anemoi-inference.git@0.8.3 + +baselines: + - baseline: + baseline_id: COSMO-E + label: COSMO-E + root: /store_new/mch/msopr/ml/COSMO-E + steps: 0/120/6 + +analysis: + label: COSMO KENDA + analysis_zarr: /scratch/mch/fzanetta/data/anemoi/datasets/mch-co2-an-archive-0p02-2015-2020-6h-v3-pl13.zarr + +stratification: + regions: + - jura + - mittelland + - voralpen + - alpennordhang + - innerealpentaeler + - alpensuedseite + root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517 + +locations: + output_root: output/ + mlflow_uri: + - https://servicedepl.meteoswiss.ch/mlstore + - https://mlflow.ecmwf.int + +profile: + executor: slurm + global_resources: + gpus: 16 + default_resources: + slurm_partition: "postproc" + cpus_per_task: 1 + mem_mb_per_cpu: 1800 + runtime: "23h" + gpus: 0 + jobs: 50 diff --git a/config/forecasters-ich1_mod_01.yaml b/config/forecasters-ich1_mod_01.yaml new file mode 100644 index 00000000..909e8180 --- /dev/null +++ b/config/forecasters-ich1_mod_01.yaml @@ -0,0 +1,57 @@ +# yaml-language-server: $schema=../workflow/tools/config.schema.json +description: | + Evaluate skill of ICON-CH1 single. + +dates: + start: 2024-01-02T00:00 + end: 2024-01-31T00:00 + frequency: 24h + +runs: + - forecaster: + mlflow_id: b30acf68520a4bbd8324c44666561696 + label: stage_C_icon_1km + steps: 0/120/6 + config: resources/inference/configs/sgm-forecaster-global-ich1.yaml + disable_local_eccodes_definitions: true + extra_dependencies: + - git+https://github.com/ecmwf/anemoi-inference.git@main + +baselines: +- baseline: + baseline_id: ICON-CH1-EPS + label: ICON-CH1-EPS + root: /store_new/mch/msopr/ml/ICON-CH1-EPS + steps: 0/33/6 + +analysis: + label: REA-L-CH1 + analysis_zarr: /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + +stratification: + regions: + - jura + - mittelland + - voralpen + - alpennordhang + - innerealpentaeler + - alpensuedseite + root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517 + +locations: + output_root: output/ + mlflow_uri: + - https://servicedepl.meteoswiss.ch/mlstore + - https://mlflow.ecmwf.int + +profile: + executor: slurm + global_resources: + gpus: 16 + default_resources: + slurm_partition: "postproc" + cpus_per_task: 1 + mem_mb_per_cpu: 1800 + runtime: "23h" + gpus: 0 + jobs: 50 diff --git a/config/forecasters-ich1_mod_02_1yr.yaml b/config/forecasters-ich1_mod_02_1yr.yaml new file mode 100644 index 00000000..e876f643 --- /dev/null +++ b/config/forecasters-ich1_mod_02_1yr.yaml @@ -0,0 +1,57 @@ +# yaml-language-server: $schema=../workflow/tools/config.schema.json +description: | + Evaluate skill of ICON-CH1 single. + +dates: + start: 2024-01-02T00:00 + end: 2024-12-25T00:00 + frequency: 24h + +runs: + - forecaster: + mlflow_id: b30acf68520a4bbd8324c44666561696 + label: stage_C_icon_1km + steps: 0/120/6 + config: resources/inference/configs/sgm-forecaster-global-ich1.yaml + disable_local_eccodes_definitions: true + extra_dependencies: + - git+https://github.com/ecmwf/anemoi-inference.git@main + +baselines: +- baseline: + baseline_id: ICON-CH1-EPS + label: ICON-CH1-EPS + root: /store_new/mch/msopr/ml/ICON-CH1-EPS + steps: 0/33/6 + +analysis: + label: REA-L-CH1 + analysis_zarr: /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + +stratification: + regions: + - jura + - mittelland + - voralpen + - alpennordhang + - innerealpentaeler + - alpensuedseite + root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517 + +locations: + output_root: output/ + mlflow_uri: + - https://servicedepl.meteoswiss.ch/mlstore + - https://mlflow.ecmwf.int + +profile: + executor: slurm + global_resources: + gpus: 16 + default_resources: + slurm_partition: "postproc" + cpus_per_task: 1 + mem_mb_per_cpu: 1800 + runtime: "23h" + gpus: 0 + jobs: 50 diff --git a/examine_error_distr_fine_tune.ipynb b/examine_error_distr_fine_tune.ipynb new file mode 100644 index 00000000..5685bb49 --- /dev/null +++ b/examine_error_distr_fine_tune.ipynb @@ -0,0 +1,597 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "b7660869", + "metadata": {}, + "source": [ + "Load Verification files and examine the distribution of the spatial components\n", + "to aid fine-tuning of the colour levels for the maps." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f38d8953", + "metadata": {}, + "outputs": [], + "source": [ + "import xarray as xr\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "eefaf693", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.DataArray 'PS.RMSE.spatial' (eps: 1, values: 1147980)> Size: 5MB\n",
+       "[1147980 values with dtype=float32]\n",
+       "Coordinates:\n",
+       "  * eps        (eps) int64 8B 0\n",
+       "    lead_time  timedelta64[ns] 8B 5 days\n",
+       "    init_hour  int64 8B 0\n",
+       "    season     <U3 12B 'all'\n",
+       "    lon        (values) float64 9MB ...\n",
+       "    lat        (values) float64 9MB ...\n",
+       "    latitude   (values) float64 9MB ...\n",
+       "    longitude  (values) float64 9MB ...\n",
+       "Dimensions without coordinates: values
" + ], + "text/plain": [ + " Size: 5MB\n", + "[1147980 values with dtype=float32]\n", + "Coordinates:\n", + " * eps (eps) int64 8B 0\n", + " lead_time timedelta64[ns] 8B 5 days\n", + " init_hour int64 8B 0\n", + " season " + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "\n", + "# 1. Define and calculate the quantiles\n", + "q_list = [0.2, 0.4, 0.6, 0.8]\n", + "rmse_quantiles = dat_sel.quantile(q_list)\n", + "\n", + "plt.figure(figsize=(12, 7))\n", + "\n", + "# 2. Plot the histogram\n", + "# We set alpha=0.6 so the vertical lines are easier to see\n", + "dat_sel.plot.hist(bins=100, color='skyblue', edgecolor='white', alpha=0.6)\n", + "\n", + "# 3. Add vertical lines for each quantile\n", + "# We'll use a colormap to give each line a different color\n", + "colors = plt.cm.viridis(np.linspace(0, 1, len(q_list)))\n", + "\n", + "for q, val, color in zip(q_list, rmse_quantiles.values, colors):\n", + " # Format the label for the legend: e.g., \"50% (Median): 4.32\"\n", + " prob_label = f\"{int(q*100)}%\"\n", + " if q == 0.5: prob_label += \" (Median)\"\n", + " \n", + " label_text = f\"{prob_label}: {val:.3f}\"\n", + " \n", + " plt.axvline(val, color=color, linestyle='--', linewidth=2, label=label_text)\n", + "\n", + "# 4. Final plot styling\n", + "plt.title(f\"Distribution of {dat_sel.name}\\nwith Statistical Quantiles\", fontsize=14)\n", + "plt.xlabel(\"RMSE Value\", fontsize=12)\n", + "plt.ylabel(\"Frequency (Grid Points)\", fontsize=12)\n", + "\n", + "# Place the legend\n", + "plt.legend(title=\"Quantiles & Values\", loc='upper right', frameon=True, shadow=True)\n", + "\n", + "plt.grid(axis='y', alpha=0.3)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "evalml (3.12.12)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/snakemake_log.txt b/snakemake_log.txt new file mode 100644 index 00000000..954cbf14 --- /dev/null +++ b/snakemake_log.txt @@ -0,0 +1,13914 @@ +Launching: snakemake --executor slurm --resources gpus=16 --default-resources slurm_partition=postproc cpus_per_task=1 mem_mb_per_cpu=1800 runtime=23h --jobs 50 --configfile config/forecasters-ich1_mod_02_1yr.yaml --cores 4 --dry-run experiment_all +host: balfrin-ln002 +Building DAG of jobs... +Job stats: +job count +---------------------------------- ------- +execute_inference 359 +experiment_all 1 +make_squashfs_image 1 +plot_summary_stat_maps 10 +plot_summary_stat_maps_baseline 10 +prepare_inference_forecaster 358 +report_experiment_dashboard 1 +verif_metrics 359 +verif_metrics_aggregation 1 +verif_metrics_aggregation_baseline 1 +verif_metrics_baseline 359 +verif_metrics_plot 1 +write_summary 1 +total 1462 + + +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402140000/config.yaml, output/data/runs/b30a-0209f019/202402140000/resources, output/data/runs/b30a-0209f019/202402140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402140000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402140000.log + jobid: 497 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401170000/config.yaml, output/data/runs/b30a-0209f019/202401170000/resources, output/data/runs/b30a-0209f019/202401170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401170000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401170000.log + jobid: 413 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407310000/config.yaml, output/data/runs/b30a-0209f019/202407310000/resources, output/data/runs/b30a-0209f019/202407310000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407310000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407310000.log + jobid: 1001 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407310000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408280000/config.yaml, output/data/runs/b30a-0209f019/202408280000/resources, output/data/runs/b30a-0209f019/202408280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408280000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408280000.log + jobid: 1085 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405180000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405180000.log + jobid: 140 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404100000/config.yaml, output/data/runs/b30a-0209f019/202404100000/resources, output/data/runs/b30a-0209f019/202404100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404100000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404100000.log + jobid: 665 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407030000/config.yaml, output/data/runs/b30a-0209f019/202407030000/resources, output/data/runs/b30a-0209f019/202407030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407030000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407030000.log + jobid: 917 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409070000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409070000.log + jobid: 252 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409250000/config.yaml, output/data/runs/b30a-0209f019/202409250000/resources, output/data/runs/b30a-0209f019/202409250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409250000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409250000.log + jobid: 1169 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410230000/config.yaml, output/data/runs/b30a-0209f019/202410230000/resources, output/data/runs/b30a-0209f019/202410230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410230000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410230000.log + jobid: 1253 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409080000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409080000.log + jobid: 253 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405190000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405190000.log + jobid: 141 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401180000/config.yaml, output/data/runs/b30a-0209f019/202401180000/resources, output/data/runs/b30a-0209f019/202401180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401180000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401180000.log + jobid: 416 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408010000/config.yaml, output/data/runs/b30a-0209f019/202408010000/resources, output/data/runs/b30a-0209f019/202408010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408010000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408010000.log + jobid: 1004 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408290000/config.yaml, output/data/runs/b30a-0209f019/202408290000/resources, output/data/runs/b30a-0209f019/202408290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408290000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408290000.log + jobid: 1088 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405200000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405200000.log + jobid: 142 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402150000/config.yaml, output/data/runs/b30a-0209f019/202402150000/resources, output/data/runs/b30a-0209f019/202402150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402150000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402150000.log + jobid: 500 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404110000/config.yaml, output/data/runs/b30a-0209f019/202404110000/resources, output/data/runs/b30a-0209f019/202404110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404110000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404110000.log + jobid: 668 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407040000/config.yaml, output/data/runs/b30a-0209f019/202407040000/resources, output/data/runs/b30a-0209f019/202407040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407040000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407040000.log + jobid: 920 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409090000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409090000.log + jobid: 254 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409260000/config.yaml, output/data/runs/b30a-0209f019/202409260000/resources, output/data/runs/b30a-0209f019/202409260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409260000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409260000.log + jobid: 1172 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410240000/config.yaml, output/data/runs/b30a-0209f019/202410240000/resources, output/data/runs/b30a-0209f019/202410240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410240000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410240000.log + jobid: 1256 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405210000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405210000.log + jobid: 143 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409100000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409100000.log + jobid: 255 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401190000/config.yaml, output/data/runs/b30a-0209f019/202401190000/resources, output/data/runs/b30a-0209f019/202401190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401190000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401190000.log + jobid: 419 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408020000/config.yaml, output/data/runs/b30a-0209f019/202408020000/resources, output/data/runs/b30a-0209f019/202408020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408020000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408020000.log + jobid: 1007 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408300000/config.yaml, output/data/runs/b30a-0209f019/202408300000/resources, output/data/runs/b30a-0209f019/202408300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408300000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408300000.log + jobid: 1091 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405220000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405220000.log + jobid: 144 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402160000/config.yaml, output/data/runs/b30a-0209f019/202402160000/resources, output/data/runs/b30a-0209f019/202402160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402160000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402160000.log + jobid: 503 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404120000/config.yaml, output/data/runs/b30a-0209f019/202404120000/resources, output/data/runs/b30a-0209f019/202404120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404120000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404120000.log + jobid: 671 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407050000/config.yaml, output/data/runs/b30a-0209f019/202407050000/resources, output/data/runs/b30a-0209f019/202407050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407050000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407050000.log + jobid: 923 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409110000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409110000.log + jobid: 256 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409270000/config.yaml, output/data/runs/b30a-0209f019/202409270000/resources, output/data/runs/b30a-0209f019/202409270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409270000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409270000.log + jobid: 1175 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410250000/config.yaml, output/data/runs/b30a-0209f019/202410250000/resources, output/data/runs/b30a-0209f019/202410250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410250000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410250000.log + jobid: 1259 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405230000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405230000.log + jobid: 145 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409120000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409120000.log + jobid: 257 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401200000/config.yaml, output/data/runs/b30a-0209f019/202401200000/resources, output/data/runs/b30a-0209f019/202401200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401200000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401200000.log + jobid: 422 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408030000/config.yaml, output/data/runs/b30a-0209f019/202408030000/resources, output/data/runs/b30a-0209f019/202408030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408030000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408030000.log + jobid: 1010 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408310000/config.yaml, output/data/runs/b30a-0209f019/202408310000/resources, output/data/runs/b30a-0209f019/202408310000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408310000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408310000.log + jobid: 1094 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408310000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405240000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405240000.log + jobid: 146 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402170000/config.yaml, output/data/runs/b30a-0209f019/202402170000/resources, output/data/runs/b30a-0209f019/202402170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402170000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402170000.log + jobid: 506 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404130000/config.yaml, output/data/runs/b30a-0209f019/202404130000/resources, output/data/runs/b30a-0209f019/202404130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404130000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404130000.log + jobid: 674 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407060000/config.yaml, output/data/runs/b30a-0209f019/202407060000/resources, output/data/runs/b30a-0209f019/202407060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407060000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407060000.log + jobid: 926 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409130000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409130000.log + jobid: 258 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409280000/config.yaml, output/data/runs/b30a-0209f019/202409280000/resources, output/data/runs/b30a-0209f019/202409280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409280000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409280000.log + jobid: 1178 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410260000/config.yaml, output/data/runs/b30a-0209f019/202410260000/resources, output/data/runs/b30a-0209f019/202410260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410260000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410260000.log + jobid: 1262 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405250000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405250000.log + jobid: 147 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409140000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409140000.log + jobid: 259 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401210000/config.yaml, output/data/runs/b30a-0209f019/202401210000/resources, output/data/runs/b30a-0209f019/202401210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401210000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401210000.log + jobid: 425 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408040000/config.yaml, output/data/runs/b30a-0209f019/202408040000/resources, output/data/runs/b30a-0209f019/202408040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408040000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408040000.log + jobid: 1013 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409010000/config.yaml, output/data/runs/b30a-0209f019/202409010000/resources, output/data/runs/b30a-0209f019/202409010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409010000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409010000.log + jobid: 1097 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule write_summary: + input: /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/config/forecasters-ich1_mod_02_1yr.yaml + output: output/data/runs/b30a-0209f019/summary.md + jobid: 1444 + reason: Set of input files has changed since last execution + wildcards: run_id=b30a-0209f019 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405260000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405260000.log + jobid: 148 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402180000/config.yaml, output/data/runs/b30a-0209f019/202402180000/resources, output/data/runs/b30a-0209f019/202402180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402180000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402180000.log + jobid: 509 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404140000/config.yaml, output/data/runs/b30a-0209f019/202404140000/resources, output/data/runs/b30a-0209f019/202404140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404140000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404140000.log + jobid: 677 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407070000/config.yaml, output/data/runs/b30a-0209f019/202407070000/resources, output/data/runs/b30a-0209f019/202407070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407070000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407070000.log + jobid: 929 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409150000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409150000.log + jobid: 260 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409290000/config.yaml, output/data/runs/b30a-0209f019/202409290000/resources, output/data/runs/b30a-0209f019/202409290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409290000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409290000.log + jobid: 1181 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405270000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405270000.log + jobid: 149 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409160000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409160000.log + jobid: 261 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408050000/config.yaml, output/data/runs/b30a-0209f019/202408050000/resources, output/data/runs/b30a-0209f019/202408050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408050000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408050000.log + jobid: 1016 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409020000/config.yaml, output/data/runs/b30a-0209f019/202409020000/resources, output/data/runs/b30a-0209f019/202409020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409020000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409020000.log + jobid: 1100 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401220000/config.yaml, output/data/runs/b30a-0209f019/202401220000/resources, output/data/runs/b30a-0209f019/202401220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401220000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401220000.log + jobid: 428 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405280000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405280000.log + jobid: 150 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402190000/config.yaml, output/data/runs/b30a-0209f019/202402190000/resources, output/data/runs/b30a-0209f019/202402190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402190000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402190000.log + jobid: 512 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404150000/config.yaml, output/data/runs/b30a-0209f019/202404150000/resources, output/data/runs/b30a-0209f019/202404150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404150000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404150000.log + jobid: 680 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407080000/config.yaml, output/data/runs/b30a-0209f019/202407080000/resources, output/data/runs/b30a-0209f019/202407080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407080000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407080000.log + jobid: 932 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409170000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409170000.log + jobid: 262 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409300000/config.yaml, output/data/runs/b30a-0209f019/202409300000/resources, output/data/runs/b30a-0209f019/202409300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409300000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409300000.log + jobid: 1184 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410280000/config.yaml, output/data/runs/b30a-0209f019/202410280000/resources, output/data/runs/b30a-0209f019/202410280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410280000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410280000.log + jobid: 1268 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405290000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405290000.log + jobid: 151 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409180000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409180000.log + jobid: 263 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401230000/config.yaml, output/data/runs/b30a-0209f019/202401230000/resources, output/data/runs/b30a-0209f019/202401230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401230000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401230000.log + jobid: 431 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408060000/config.yaml, output/data/runs/b30a-0209f019/202408060000/resources, output/data/runs/b30a-0209f019/202408060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408060000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408060000.log + jobid: 1019 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409030000/config.yaml, output/data/runs/b30a-0209f019/202409030000/resources, output/data/runs/b30a-0209f019/202409030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409030000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409030000.log + jobid: 1103 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405300000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405300000.log + jobid: 152 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402200000/config.yaml, output/data/runs/b30a-0209f019/202402200000/resources, output/data/runs/b30a-0209f019/202402200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402200000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402200000.log + jobid: 515 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404160000/config.yaml, output/data/runs/b30a-0209f019/202404160000/resources, output/data/runs/b30a-0209f019/202404160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404160000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404160000.log + jobid: 683 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407090000/config.yaml, output/data/runs/b30a-0209f019/202407090000/resources, output/data/runs/b30a-0209f019/202407090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407090000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407090000.log + jobid: 935 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409190000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409190000.log + jobid: 264 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410010000/config.yaml, output/data/runs/b30a-0209f019/202410010000/resources, output/data/runs/b30a-0209f019/202410010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410010000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410010000.log + jobid: 1187 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410290000/config.yaml, output/data/runs/b30a-0209f019/202410290000/resources, output/data/runs/b30a-0209f019/202410290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410290000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410290000.log + jobid: 1271 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405310000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405310000.log + jobid: 153 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405310000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405310000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409200000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409200000.log + jobid: 265 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401240000/config.yaml, output/data/runs/b30a-0209f019/202401240000/resources, output/data/runs/b30a-0209f019/202401240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401240000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401240000.log + jobid: 434 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408070000/config.yaml, output/data/runs/b30a-0209f019/202408070000/resources, output/data/runs/b30a-0209f019/202408070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408070000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408070000.log + jobid: 1022 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409040000/config.yaml, output/data/runs/b30a-0209f019/202409040000/resources, output/data/runs/b30a-0209f019/202409040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409040000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409040000.log + jobid: 1106 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406010000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406010000.log + jobid: 154 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402210000/config.yaml, output/data/runs/b30a-0209f019/202402210000/resources, output/data/runs/b30a-0209f019/202402210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402210000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402210000.log + jobid: 518 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407100000/config.yaml, output/data/runs/b30a-0209f019/202407100000/resources, output/data/runs/b30a-0209f019/202407100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407100000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407100000.log + jobid: 938 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404170000/config.yaml, output/data/runs/b30a-0209f019/202404170000/resources, output/data/runs/b30a-0209f019/202404170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404170000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404170000.log + jobid: 686 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409210000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409210000.log + jobid: 266 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410020000/config.yaml, output/data/runs/b30a-0209f019/202410020000/resources, output/data/runs/b30a-0209f019/202410020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410020000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410020000.log + jobid: 1190 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410300000/config.yaml, output/data/runs/b30a-0209f019/202410300000/resources, output/data/runs/b30a-0209f019/202410300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410300000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410300000.log + jobid: 1274 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406020000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406020000.log + jobid: 155 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409220000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409220000.log + jobid: 267 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401250000/config.yaml, output/data/runs/b30a-0209f019/202401250000/resources, output/data/runs/b30a-0209f019/202401250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401250000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401250000.log + jobid: 437 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408080000/config.yaml, output/data/runs/b30a-0209f019/202408080000/resources, output/data/runs/b30a-0209f019/202408080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408080000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408080000.log + jobid: 1025 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409050000/config.yaml, output/data/runs/b30a-0209f019/202409050000/resources, output/data/runs/b30a-0209f019/202409050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409050000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409050000.log + jobid: 1109 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406030000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406030000.log + jobid: 156 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402220000/config.yaml, output/data/runs/b30a-0209f019/202402220000/resources, output/data/runs/b30a-0209f019/202402220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402220000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402220000.log + jobid: 521 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404180000/config.yaml, output/data/runs/b30a-0209f019/202404180000/resources, output/data/runs/b30a-0209f019/202404180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404180000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404180000.log + jobid: 689 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407110000/config.yaml, output/data/runs/b30a-0209f019/202407110000/resources, output/data/runs/b30a-0209f019/202407110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407110000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407110000.log + jobid: 941 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:23 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409230000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409230000.log + jobid: 268 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410030000/config.yaml, output/data/runs/b30a-0209f019/202410030000/resources, output/data/runs/b30a-0209f019/202410030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410030000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410030000.log + jobid: 1193 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410310000/config.yaml, output/data/runs/b30a-0209f019/202410310000/resources, output/data/runs/b30a-0209f019/202410310000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410310000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410310000.log + jobid: 1277 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410310000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406040000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406040000.log + jobid: 157 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409240000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409240000.log + jobid: 269 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401260000/config.yaml, output/data/runs/b30a-0209f019/202401260000/resources, output/data/runs/b30a-0209f019/202401260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401260000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401260000.log + jobid: 440 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408090000/config.yaml, output/data/runs/b30a-0209f019/202408090000/resources, output/data/runs/b30a-0209f019/202408090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408090000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408090000.log + jobid: 1028 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409060000/config.yaml, output/data/runs/b30a-0209f019/202409060000/resources, output/data/runs/b30a-0209f019/202409060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409060000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409060000.log + jobid: 1112 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406050000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406050000.log + jobid: 158 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402230000/config.yaml, output/data/runs/b30a-0209f019/202402230000/resources, output/data/runs/b30a-0209f019/202402230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402230000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402230000.log + jobid: 524 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404190000/config.yaml, output/data/runs/b30a-0209f019/202404190000/resources, output/data/runs/b30a-0209f019/202404190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404190000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404190000.log + jobid: 692 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407120000/config.yaml, output/data/runs/b30a-0209f019/202407120000/resources, output/data/runs/b30a-0209f019/202407120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407120000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407120000.log + jobid: 944 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409250000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409250000.log + jobid: 270 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410040000/config.yaml, output/data/runs/b30a-0209f019/202410040000/resources, output/data/runs/b30a-0209f019/202410040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410040000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410040000.log + jobid: 1196 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411010000/config.yaml, output/data/runs/b30a-0209f019/202411010000/resources, output/data/runs/b30a-0209f019/202411010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411010000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411010000.log + jobid: 1280 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406060000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406060000.log + jobid: 159 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409260000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409260000.log + jobid: 271 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401270000/config.yaml, output/data/runs/b30a-0209f019/202401270000/resources, output/data/runs/b30a-0209f019/202401270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401270000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401270000.log + jobid: 443 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409070000/config.yaml, output/data/runs/b30a-0209f019/202409070000/resources, output/data/runs/b30a-0209f019/202409070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409070000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409070000.log + jobid: 1115 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408100000/config.yaml, output/data/runs/b30a-0209f019/202408100000/resources, output/data/runs/b30a-0209f019/202408100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408100000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408100000.log + jobid: 1031 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406070000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406070000.log + jobid: 160 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402240000/config.yaml, output/data/runs/b30a-0209f019/202402240000/resources, output/data/runs/b30a-0209f019/202402240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402240000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402240000.log + jobid: 527 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404200000/config.yaml, output/data/runs/b30a-0209f019/202404200000/resources, output/data/runs/b30a-0209f019/202404200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404200000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404200000.log + jobid: 695 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407130000/config.yaml, output/data/runs/b30a-0209f019/202407130000/resources, output/data/runs/b30a-0209f019/202407130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407130000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407130000.log + jobid: 947 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409270000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409270000.log + jobid: 272 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410050000/config.yaml, output/data/runs/b30a-0209f019/202410050000/resources, output/data/runs/b30a-0209f019/202410050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410050000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410050000.log + jobid: 1199 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411020000/config.yaml, output/data/runs/b30a-0209f019/202411020000/resources, output/data/runs/b30a-0209f019/202411020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411020000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411020000.log + jobid: 1283 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406080000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406080000.log + jobid: 161 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409280000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409280000.log + jobid: 273 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401280000/config.yaml, output/data/runs/b30a-0209f019/202401280000/resources, output/data/runs/b30a-0209f019/202401280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401280000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401280000.log + jobid: 446 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409080000/config.yaml, output/data/runs/b30a-0209f019/202409080000/resources, output/data/runs/b30a-0209f019/202409080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409080000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409080000.log + jobid: 1118 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408110000/config.yaml, output/data/runs/b30a-0209f019/202408110000/resources, output/data/runs/b30a-0209f019/202408110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408110000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408110000.log + jobid: 1034 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406090000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406090000.log + jobid: 162 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402250000/config.yaml, output/data/runs/b30a-0209f019/202402250000/resources, output/data/runs/b30a-0209f019/202402250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402250000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402250000.log + jobid: 530 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404210000/config.yaml, output/data/runs/b30a-0209f019/202404210000/resources, output/data/runs/b30a-0209f019/202404210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404210000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404210000.log + jobid: 698 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407140000/config.yaml, output/data/runs/b30a-0209f019/202407140000/resources, output/data/runs/b30a-0209f019/202407140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407140000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407140000.log + jobid: 950 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule make_squashfs_image: + input: output/data/runs/b30a-0209f019/.venv + output: output/data/runs/b30a-0209f019/venv.squashfs + log: output/logs/make_squashfs_image/b30a-0209f019.log + jobid: 367 + reason: Missing output files: output/data/runs/b30a-0209f019/venv.squashfs; Updated input files: output/data/runs/b30a-0209f019/.venv + wildcards: run_id=b30a-0209f019 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409290000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409290000.log + jobid: 274 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410060000/config.yaml, output/data/runs/b30a-0209f019/202410060000/resources, output/data/runs/b30a-0209f019/202410060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410060000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410060000.log + jobid: 1202 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406100000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406100000.log + jobid: 163 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409300000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409300000.log + jobid: 275 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408120000/config.yaml, output/data/runs/b30a-0209f019/202408120000/resources, output/data/runs/b30a-0209f019/202408120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408120000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408120000.log + jobid: 1037 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401290000/config.yaml, output/data/runs/b30a-0209f019/202401290000/resources, output/data/runs/b30a-0209f019/202401290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401290000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401290000.log + jobid: 449 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409090000/config.yaml, output/data/runs/b30a-0209f019/202409090000/resources, output/data/runs/b30a-0209f019/202409090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409090000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409090000.log + jobid: 1121 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406110000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406110000.log + jobid: 164 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402260000/config.yaml, output/data/runs/b30a-0209f019/202402260000/resources, output/data/runs/b30a-0209f019/202402260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402260000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402260000.log + jobid: 533 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404220000/config.yaml, output/data/runs/b30a-0209f019/202404220000/resources, output/data/runs/b30a-0209f019/202404220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404220000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404220000.log + jobid: 701 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407150000/config.yaml, output/data/runs/b30a-0209f019/202407150000/resources, output/data/runs/b30a-0209f019/202407150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407150000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407150000.log + jobid: 953 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410010000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410010000.log + jobid: 276 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410070000/config.yaml, output/data/runs/b30a-0209f019/202410070000/resources, output/data/runs/b30a-0209f019/202410070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410070000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410070000.log + jobid: 1205 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411040000/config.yaml, output/data/runs/b30a-0209f019/202411040000/resources, output/data/runs/b30a-0209f019/202411040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411040000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411040000.log + jobid: 1289 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406120000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406120000.log + jobid: 165 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410020000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410020000.log + jobid: 277 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408130000/config.yaml, output/data/runs/b30a-0209f019/202408130000/resources, output/data/runs/b30a-0209f019/202408130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408130000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408130000.log + jobid: 1040 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401300000/config.yaml, output/data/runs/b30a-0209f019/202401300000/resources, output/data/runs/b30a-0209f019/202401300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401300000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401300000.log + jobid: 452 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409100000/config.yaml, output/data/runs/b30a-0209f019/202409100000/resources, output/data/runs/b30a-0209f019/202409100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409100000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409100000.log + jobid: 1124 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406130000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406130000.log + jobid: 166 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402270000/config.yaml, output/data/runs/b30a-0209f019/202402270000/resources, output/data/runs/b30a-0209f019/202402270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402270000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402270000.log + jobid: 536 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404230000/config.yaml, output/data/runs/b30a-0209f019/202404230000/resources, output/data/runs/b30a-0209f019/202404230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404230000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404230000.log + jobid: 704 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407160000/config.yaml, output/data/runs/b30a-0209f019/202407160000/resources, output/data/runs/b30a-0209f019/202407160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407160000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407160000.log + jobid: 956 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410030000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410030000.log + jobid: 278 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410080000/config.yaml, output/data/runs/b30a-0209f019/202410080000/resources, output/data/runs/b30a-0209f019/202410080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410080000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410080000.log + jobid: 1208 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411050000/config.yaml, output/data/runs/b30a-0209f019/202411050000/resources, output/data/runs/b30a-0209f019/202411050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411050000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411050000.log + jobid: 1292 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406140000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406140000.log + jobid: 167 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410040000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410040000.log + jobid: 279 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402280000/config.yaml, output/data/runs/b30a-0209f019/202402280000/resources, output/data/runs/b30a-0209f019/202402280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402280000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402280000.log + jobid: 539 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404240000/config.yaml, output/data/runs/b30a-0209f019/202404240000/resources, output/data/runs/b30a-0209f019/202404240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404240000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404240000.log + jobid: 707 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407170000/config.yaml, output/data/runs/b30a-0209f019/202407170000/resources, output/data/runs/b30a-0209f019/202407170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407170000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407170000.log + jobid: 959 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401310000/config.yaml, output/data/runs/b30a-0209f019/202401310000/resources, output/data/runs/b30a-0209f019/202401310000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401310000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401310000.log + jobid: 455 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401310000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406150000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406150000.log + jobid: 168 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401030000/config.yaml, output/data/runs/b30a-0209f019/202401030000/resources, output/data/runs/b30a-0209f019/202401030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401030000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401030000.log + jobid: 371 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409110000/config.yaml, output/data/runs/b30a-0209f019/202409110000/resources, output/data/runs/b30a-0209f019/202409110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409110000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409110000.log + jobid: 1127 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411060000/config.yaml, output/data/runs/b30a-0209f019/202411060000/resources, output/data/runs/b30a-0209f019/202411060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411060000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411060000.log + jobid: 1295 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410050000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410050000.log + jobid: 280 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408140000/config.yaml, output/data/runs/b30a-0209f019/202408140000/resources, output/data/runs/b30a-0209f019/202408140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408140000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408140000.log + jobid: 1043 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406160000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406160000.log + jobid: 169 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410060000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410060000.log + jobid: 281 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402010000/config.yaml, output/data/runs/b30a-0209f019/202402010000/resources, output/data/runs/b30a-0209f019/202402010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402010000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402010000.log + jobid: 458 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408150000/config.yaml, output/data/runs/b30a-0209f019/202408150000/resources, output/data/runs/b30a-0209f019/202408150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408150000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408150000.log + jobid: 1046 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409120000/config.yaml, output/data/runs/b30a-0209f019/202409120000/resources, output/data/runs/b30a-0209f019/202409120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409120000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409120000.log + jobid: 1130 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406170000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406170000.log + jobid: 170 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402290000/config.yaml, output/data/runs/b30a-0209f019/202402290000/resources, output/data/runs/b30a-0209f019/202402290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402290000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402290000.log + jobid: 542 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401040000/config.yaml, output/data/runs/b30a-0209f019/202401040000/resources, output/data/runs/b30a-0209f019/202401040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401040000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401040000.log + jobid: 374 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404250000/config.yaml, output/data/runs/b30a-0209f019/202404250000/resources, output/data/runs/b30a-0209f019/202404250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404250000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404250000.log + jobid: 710 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407180000/config.yaml, output/data/runs/b30a-0209f019/202407180000/resources, output/data/runs/b30a-0209f019/202407180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407180000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407180000.log + jobid: 962 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410070000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410070000.log + jobid: 282 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411070000/config.yaml, output/data/runs/b30a-0209f019/202411070000/resources, output/data/runs/b30a-0209f019/202411070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411070000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411070000.log + jobid: 1298 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401020000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401020000.log + jobid: 3 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406180000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406180000.log + jobid: 171 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410080000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410080000.log + jobid: 283 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409130000/config.yaml, output/data/runs/b30a-0209f019/202409130000/resources, output/data/runs/b30a-0209f019/202409130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409130000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409130000.log + jobid: 1133 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402020000/config.yaml, output/data/runs/b30a-0209f019/202402020000/resources, output/data/runs/b30a-0209f019/202402020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402020000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402020000.log + jobid: 461 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408160000/config.yaml, output/data/runs/b30a-0209f019/202408160000/resources, output/data/runs/b30a-0209f019/202408160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408160000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408160000.log + jobid: 1049 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406190000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406190000.log + jobid: 172 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401030000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401030000.log + jobid: 4 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403010000/config.yaml, output/data/runs/b30a-0209f019/202403010000/resources, output/data/runs/b30a-0209f019/202403010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403010000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403010000.log + jobid: 545 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401050000/config.yaml, output/data/runs/b30a-0209f019/202401050000/resources, output/data/runs/b30a-0209f019/202401050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401050000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401050000.log + jobid: 377 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404260000/config.yaml, output/data/runs/b30a-0209f019/202404260000/resources, output/data/runs/b30a-0209f019/202404260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404260000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404260000.log + jobid: 713 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410090000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410090000.log + jobid: 284 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411080000/config.yaml, output/data/runs/b30a-0209f019/202411080000/resources, output/data/runs/b30a-0209f019/202411080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411080000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411080000.log + jobid: 1301 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406200000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406200000.log + jobid: 173 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401040000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401040000.log + jobid: 5 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410100000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410100000.log + jobid: 285 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403020000/config.yaml, output/data/runs/b30a-0209f019/202403020000/resources, output/data/runs/b30a-0209f019/202403020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403020000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403020000.log + jobid: 548 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409140000/config.yaml, output/data/runs/b30a-0209f019/202409140000/resources, output/data/runs/b30a-0209f019/202409140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409140000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409140000.log + jobid: 1136 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402030000/config.yaml, output/data/runs/b30a-0209f019/202402030000/resources, output/data/runs/b30a-0209f019/202402030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402030000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402030000.log + jobid: 464 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408170000/config.yaml, output/data/runs/b30a-0209f019/202408170000/resources, output/data/runs/b30a-0209f019/202408170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408170000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408170000.log + jobid: 1052 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406210000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406210000.log + jobid: 174 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401050000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401050000.log + jobid: 6 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404270000/config.yaml, output/data/runs/b30a-0209f019/202404270000/resources, output/data/runs/b30a-0209f019/202404270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404270000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404270000.log + jobid: 716 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407200000/config.yaml, output/data/runs/b30a-0209f019/202407200000/resources, output/data/runs/b30a-0209f019/202407200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407200000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407200000.log + jobid: 968 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401060000/config.yaml, output/data/runs/b30a-0209f019/202401060000/resources, output/data/runs/b30a-0209f019/202401060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401060000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401060000.log + jobid: 380 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410110000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410110000.log + jobid: 286 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406220000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406220000.log + jobid: 175 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401060000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401060000.log + jobid: 7 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410120000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410120000.log + jobid: 287 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409150000/config.yaml, output/data/runs/b30a-0209f019/202409150000/resources, output/data/runs/b30a-0209f019/202409150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409150000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409150000.log + jobid: 1139 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402040000/config.yaml, output/data/runs/b30a-0209f019/202402040000/resources, output/data/runs/b30a-0209f019/202402040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402040000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402040000.log + jobid: 467 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408180000/config.yaml, output/data/runs/b30a-0209f019/202408180000/resources, output/data/runs/b30a-0209f019/202408180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408180000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408180000.log + jobid: 1055 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401070000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401070000.log + jobid: 8 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406230000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406230000.log + jobid: 176 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403030000/config.yaml, output/data/runs/b30a-0209f019/202403030000/resources, output/data/runs/b30a-0209f019/202403030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403030000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403030000.log + jobid: 551 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404280000/config.yaml, output/data/runs/b30a-0209f019/202404280000/resources, output/data/runs/b30a-0209f019/202404280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404280000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404280000.log + jobid: 719 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407210000/config.yaml, output/data/runs/b30a-0209f019/202407210000/resources, output/data/runs/b30a-0209f019/202407210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407210000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407210000.log + jobid: 971 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410130000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410130000.log + jobid: 288 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410130000/config.yaml, output/data/runs/b30a-0209f019/202410130000/resources, output/data/runs/b30a-0209f019/202410130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410130000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410130000.log + jobid: 1223 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401080000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401080000.log + jobid: 9 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406240000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406240000.log + jobid: 177 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401070000/config.yaml, output/data/runs/b30a-0209f019/202401070000/resources, output/data/runs/b30a-0209f019/202401070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401070000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401070000.log + jobid: 383 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410140000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410140000.log + jobid: 289 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410270000/config.yaml, output/data/runs/b30a-0209f019/202410270000/resources, output/data/runs/b30a-0209f019/202410270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410270000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410270000.log + jobid: 1265 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409160000/config.yaml, output/data/runs/b30a-0209f019/202409160000/resources, output/data/runs/b30a-0209f019/202409160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409160000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409160000.log + jobid: 1142 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402050000/config.yaml, output/data/runs/b30a-0209f019/202402050000/resources, output/data/runs/b30a-0209f019/202402050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402050000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402050000.log + jobid: 470 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408190000/config.yaml, output/data/runs/b30a-0209f019/202408190000/resources, output/data/runs/b30a-0209f019/202408190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408190000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408190000.log + jobid: 1058 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401090000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401090000.log + jobid: 10 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406250000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406250000.log + jobid: 178 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403040000/config.yaml, output/data/runs/b30a-0209f019/202403040000/resources, output/data/runs/b30a-0209f019/202403040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403040000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403040000.log + jobid: 554 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404290000/config.yaml, output/data/runs/b30a-0209f019/202404290000/resources, output/data/runs/b30a-0209f019/202404290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404290000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404290000.log + jobid: 722 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407220000/config.yaml, output/data/runs/b30a-0209f019/202407220000/resources, output/data/runs/b30a-0209f019/202407220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407220000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407220000.log + jobid: 974 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410150000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410150000.log + jobid: 290 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410140000/config.yaml, output/data/runs/b30a-0209f019/202410140000/resources, output/data/runs/b30a-0209f019/202410140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410140000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410140000.log + jobid: 1226 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401100000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401100000.log + jobid: 11 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406260000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406260000.log + jobid: 179 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410160000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410160000.log + jobid: 291 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410170000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410170000.log + jobid: 292 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403050000/config.yaml, output/data/runs/b30a-0209f019/202403050000/resources, output/data/runs/b30a-0209f019/202403050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403050000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403050000.log + jobid: 557 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402060000/config.yaml, output/data/runs/b30a-0209f019/202402060000/resources, output/data/runs/b30a-0209f019/202402060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402060000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402060000.log + jobid: 473 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409170000/config.yaml, output/data/runs/b30a-0209f019/202409170000/resources, output/data/runs/b30a-0209f019/202409170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409170000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409170000.log + jobid: 1145 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408200000/config.yaml, output/data/runs/b30a-0209f019/202408200000/resources, output/data/runs/b30a-0209f019/202408200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408200000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408200000.log + jobid: 1061 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401110000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401110000.log + jobid: 12 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406270000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406270000.log + jobid: 180 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404300000/config.yaml, output/data/runs/b30a-0209f019/202404300000/resources, output/data/runs/b30a-0209f019/202404300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404300000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404300000.log + jobid: 725 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407230000/config.yaml, output/data/runs/b30a-0209f019/202407230000/resources, output/data/runs/b30a-0209f019/202407230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407230000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407230000.log + jobid: 977 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410150000/config.yaml, output/data/runs/b30a-0209f019/202410150000/resources, output/data/runs/b30a-0209f019/202410150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410150000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410150000.log + jobid: 1229 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401120000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401120000.log + jobid: 13 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406280000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406280000.log + jobid: 181 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401080000/config.yaml, output/data/runs/b30a-0209f019/202401080000/resources, output/data/runs/b30a-0209f019/202401080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401080000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401080000.log + jobid: 386 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410180000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410180000.log + jobid: 293 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410190000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410190000.log + jobid: 294 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408210000/config.yaml, output/data/runs/b30a-0209f019/202408210000/resources, output/data/runs/b30a-0209f019/202408210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408210000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408210000.log + jobid: 1064 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402070000/config.yaml, output/data/runs/b30a-0209f019/202402070000/resources, output/data/runs/b30a-0209f019/202402070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402070000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402070000.log + jobid: 476 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409180000/config.yaml, output/data/runs/b30a-0209f019/202409180000/resources, output/data/runs/b30a-0209f019/202409180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409180000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409180000.log + jobid: 1148 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401130000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401130000.log + jobid: 14 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406290000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406290000.log + jobid: 182 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403060000/config.yaml, output/data/runs/b30a-0209f019/202403060000/resources, output/data/runs/b30a-0209f019/202403060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403060000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403060000.log + jobid: 560 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407240000/config.yaml, output/data/runs/b30a-0209f019/202407240000/resources, output/data/runs/b30a-0209f019/202407240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407240000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407240000.log + jobid: 980 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405010000/config.yaml, output/data/runs/b30a-0209f019/202405010000/resources, output/data/runs/b30a-0209f019/202405010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405010000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405010000.log + jobid: 728 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410160000/config.yaml, output/data/runs/b30a-0209f019/202410160000/resources, output/data/runs/b30a-0209f019/202410160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410160000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410160000.log + jobid: 1232 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401140000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401140000.log + jobid: 15 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202406300000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202406300000.log + jobid: 183 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202406300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202406300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401090000/config.yaml, output/data/runs/b30a-0209f019/202401090000/resources, output/data/runs/b30a-0209f019/202401090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401090000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401090000.log + jobid: 389 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410200000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410200000.log + jobid: 295 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407010000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407010000.log + jobid: 184 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402080000/config.yaml, output/data/runs/b30a-0209f019/202402080000/resources, output/data/runs/b30a-0209f019/202402080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402080000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402080000.log + jobid: 479 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409190000/config.yaml, output/data/runs/b30a-0209f019/202409190000/resources, output/data/runs/b30a-0209f019/202409190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409190000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409190000.log + jobid: 1151 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408220000/config.yaml, output/data/runs/b30a-0209f019/202408220000/resources, output/data/runs/b30a-0209f019/202408220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408220000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408220000.log + jobid: 1067 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401150000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401150000.log + jobid: 16 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403070000/config.yaml, output/data/runs/b30a-0209f019/202403070000/resources, output/data/runs/b30a-0209f019/202403070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403070000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403070000.log + jobid: 563 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405020000/config.yaml, output/data/runs/b30a-0209f019/202405020000/resources, output/data/runs/b30a-0209f019/202405020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405020000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405020000.log + jobid: 731 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407250000/config.yaml, output/data/runs/b30a-0209f019/202407250000/resources, output/data/runs/b30a-0209f019/202407250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407250000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407250000.log + jobid: 983 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410170000/config.yaml, output/data/runs/b30a-0209f019/202410170000/resources, output/data/runs/b30a-0209f019/202410170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410170000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410170000.log + jobid: 1235 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410210000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410210000.log + jobid: 296 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401160000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401160000.log + jobid: 17 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407020000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407020000.log + jobid: 185 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401100000/config.yaml, output/data/runs/b30a-0209f019/202401100000/resources, output/data/runs/b30a-0209f019/202401100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401100000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401100000.log + jobid: 392 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410220000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410220000.log + jobid: 297 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411150000/config.yaml, output/data/runs/b30a-0209f019/202411150000/resources, output/data/runs/b30a-0209f019/202411150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411150000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411150000.log + jobid: 1322 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407260000/config.yaml, output/data/runs/b30a-0209f019/202407260000/resources, output/data/runs/b30a-0209f019/202407260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407260000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407260000.log + jobid: 986 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410180000/config.yaml, output/data/runs/b30a-0209f019/202410180000/resources, output/data/runs/b30a-0209f019/202410180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410180000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410180000.log + jobid: 1238 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403080000/config.yaml, output/data/runs/b30a-0209f019/202403080000/resources, output/data/runs/b30a-0209f019/202403080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403080000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403080000.log + jobid: 566 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408230000/config.yaml, output/data/runs/b30a-0209f019/202408230000/resources, output/data/runs/b30a-0209f019/202408230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408230000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408230000.log + jobid: 1070 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402090000/config.yaml, output/data/runs/b30a-0209f019/202402090000/resources, output/data/runs/b30a-0209f019/202402090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402090000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402090000.log + jobid: 482 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409200000/config.yaml, output/data/runs/b30a-0209f019/202409200000/resources, output/data/runs/b30a-0209f019/202409200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409200000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409200000.log + jobid: 1154 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401170000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401170000.log + jobid: 18 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407030000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407030000.log + jobid: 186 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405030000/config.yaml, output/data/runs/b30a-0209f019/202405030000/resources, output/data/runs/b30a-0209f019/202405030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405030000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405030000.log + jobid: 734 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401180000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401180000.log + jobid: 19 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407040000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407040000.log + jobid: 187 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401110000/config.yaml, output/data/runs/b30a-0209f019/202401110000/resources, output/data/runs/b30a-0209f019/202401110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401110000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401110000.log + jobid: 395 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410240000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410240000.log + jobid: 299 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403090000/config.yaml, output/data/runs/b30a-0209f019/202403090000/resources, output/data/runs/b30a-0209f019/202403090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403090000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403090000.log + jobid: 569 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408240000/config.yaml, output/data/runs/b30a-0209f019/202408240000/resources, output/data/runs/b30a-0209f019/202408240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408240000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408240000.log + jobid: 1073 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402100000/config.yaml, output/data/runs/b30a-0209f019/202402100000/resources, output/data/runs/b30a-0209f019/202402100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402100000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402100000.log + jobid: 485 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409210000/config.yaml, output/data/runs/b30a-0209f019/202409210000/resources, output/data/runs/b30a-0209f019/202409210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409210000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409210000.log + jobid: 1157 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401190000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401190000.log + jobid: 20 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407050000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407050000.log + jobid: 188 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401120000/config.yaml, output/data/runs/b30a-0209f019/202401120000/resources, output/data/runs/b30a-0209f019/202401120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401120000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401120000.log + jobid: 398 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405040000/config.yaml, output/data/runs/b30a-0209f019/202405040000/resources, output/data/runs/b30a-0209f019/202405040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405040000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405040000.log + jobid: 737 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407270000/config.yaml, output/data/runs/b30a-0209f019/202407270000/resources, output/data/runs/b30a-0209f019/202407270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407270000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407270000.log + jobid: 989 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411160000/config.yaml, output/data/runs/b30a-0209f019/202411160000/resources, output/data/runs/b30a-0209f019/202411160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411160000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411160000.log + jobid: 1325 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401200000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401200000.log + jobid: 21 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407060000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407060000.log + jobid: 189 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410260000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410260000.log + jobid: 301 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403100000/config.yaml, output/data/runs/b30a-0209f019/202403100000/resources, output/data/runs/b30a-0209f019/202403100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403100000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403100000.log + jobid: 572 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408250000/config.yaml, output/data/runs/b30a-0209f019/202408250000/resources, output/data/runs/b30a-0209f019/202408250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408250000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408250000.log + jobid: 1076 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402110000/config.yaml, output/data/runs/b30a-0209f019/202402110000/resources, output/data/runs/b30a-0209f019/202402110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402110000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402110000.log + jobid: 488 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409220000/config.yaml, output/data/runs/b30a-0209f019/202409220000/resources, output/data/runs/b30a-0209f019/202409220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409220000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409220000.log + jobid: 1160 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401210000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401210000.log + jobid: 22 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407070000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407070000.log + jobid: 190 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407280000/config.yaml, output/data/runs/b30a-0209f019/202407280000/resources, output/data/runs/b30a-0209f019/202407280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407280000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407280000.log + jobid: 992 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401140000/config.yaml, output/data/runs/b30a-0209f019/202401140000/resources, output/data/runs/b30a-0209f019/202401140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401140000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401140000.log + jobid: 404 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405050000/config.yaml, output/data/runs/b30a-0209f019/202405050000/resources, output/data/runs/b30a-0209f019/202405050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405050000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405050000.log + jobid: 740 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411170000/config.yaml, output/data/runs/b30a-0209f019/202411170000/resources, output/data/runs/b30a-0209f019/202411170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411170000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411170000.log + jobid: 1328 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401220000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401220000.log + jobid: 23 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407080000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407080000.log + jobid: 191 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410280000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410280000.log + jobid: 303 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403110000/config.yaml, output/data/runs/b30a-0209f019/202403110000/resources, output/data/runs/b30a-0209f019/202403110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403110000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403110000.log + jobid: 575 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408260000/config.yaml, output/data/runs/b30a-0209f019/202408260000/resources, output/data/runs/b30a-0209f019/202408260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408260000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408260000.log + jobid: 1079 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402120000/config.yaml, output/data/runs/b30a-0209f019/202402120000/resources, output/data/runs/b30a-0209f019/202402120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402120000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402120000.log + jobid: 491 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409230000/config.yaml, output/data/runs/b30a-0209f019/202409230000/resources, output/data/runs/b30a-0209f019/202409230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409230000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409230000.log + jobid: 1163 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401230000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401230000.log + jobid: 24 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407090000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407090000.log + jobid: 192 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405060000/config.yaml, output/data/runs/b30a-0209f019/202405060000/resources, output/data/runs/b30a-0209f019/202405060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405060000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405060000.log + jobid: 743 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407290000/config.yaml, output/data/runs/b30a-0209f019/202407290000/resources, output/data/runs/b30a-0209f019/202407290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407290000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407290000.log + jobid: 995 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401150000/config.yaml, output/data/runs/b30a-0209f019/202401150000/resources, output/data/runs/b30a-0209f019/202401150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401150000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401150000.log + jobid: 407 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411180000/config.yaml, output/data/runs/b30a-0209f019/202411180000/resources, output/data/runs/b30a-0209f019/202411180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411180000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411180000.log + jobid: 1331 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401240000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401240000.log + jobid: 25 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407100000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407100000.log + jobid: 193 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410300000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410300000.log + jobid: 305 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411190000/config.yaml, output/data/runs/b30a-0209f019/202411190000/resources, output/data/runs/b30a-0209f019/202411190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411190000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411190000.log + jobid: 1334 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407110000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407110000.log + jobid: 194 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202409240000/config.yaml, output/data/runs/b30a-0209f019/202409240000/resources, output/data/runs/b30a-0209f019/202409240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202409240000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202409240000.log + jobid: 1166 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202409240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202408270000/config.yaml, output/data/runs/b30a-0209f019/202408270000/resources, output/data/runs/b30a-0209f019/202408270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202408270000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202408270000.log + jobid: 1082 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202408270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202402130000/config.yaml, output/data/runs/b30a-0209f019/202402130000/resources, output/data/runs/b30a-0209f019/202402130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202402130000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202402130000.log + jobid: 494 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202402130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403120000/config.yaml, output/data/runs/b30a-0209f019/202403120000/resources, output/data/runs/b30a-0209f019/202403120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403120000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403120000.log + jobid: 578 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401250000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401250000.log + jobid: 26 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405070000/config.yaml, output/data/runs/b30a-0209f019/202405070000/resources, output/data/runs/b30a-0209f019/202405070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405070000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405070000.log + jobid: 746 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407300000/config.yaml, output/data/runs/b30a-0209f019/202407300000/resources, output/data/runs/b30a-0209f019/202407300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407300000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407300000.log + jobid: 998 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401160000/config.yaml, output/data/runs/b30a-0209f019/202401160000/resources, output/data/runs/b30a-0209f019/202401160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401160000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401160000.log + jobid: 410 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410190000/config.yaml, output/data/runs/b30a-0209f019/202410190000/resources, output/data/runs/b30a-0209f019/202410190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410190000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410190000.log + jobid: 1241 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411090000/config.yaml, output/data/runs/b30a-0209f019/202411090000/resources, output/data/runs/b30a-0209f019/202411090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411090000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411090000.log + jobid: 1304 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401260000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401260000.log + jobid: 27 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407120000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407120000.log + jobid: 195 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411010000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411010000.log + jobid: 307 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403130000/config.yaml, output/data/runs/b30a-0209f019/202403130000/resources, output/data/runs/b30a-0209f019/202403130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403130000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403130000.log + jobid: 581 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405080000/config.yaml, output/data/runs/b30a-0209f019/202405080000/resources, output/data/runs/b30a-0209f019/202405080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405080000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405080000.log + jobid: 749 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411200000/config.yaml, output/data/runs/b30a-0209f019/202411200000/resources, output/data/runs/b30a-0209f019/202411200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411200000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411200000.log + jobid: 1337 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407130000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407130000.log + jobid: 196 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406050000/config.yaml, output/data/runs/b30a-0209f019/202406050000/resources, output/data/runs/b30a-0209f019/202406050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406050000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406050000.log + jobid: 833 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412180000/config.yaml, output/data/runs/b30a-0209f019/202412180000/resources, output/data/runs/b30a-0209f019/202412180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412180000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412180000.log + jobid: 1421 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401270000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401270000.log + jobid: 28 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403230000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403230000.log + jobid: 84 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411020000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411020000.log + jobid: 308 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407140000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407140000.log + jobid: 197 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401280000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401280000.log + jobid: 29 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403240000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403240000.log + jobid: 85 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411030000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411030000.log + jobid: 309 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410230000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410230000.log + jobid: 298 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403140000/config.yaml, output/data/runs/b30a-0209f019/202403140000/resources, output/data/runs/b30a-0209f019/202403140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403140000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403140000.log + jobid: 584 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405090000/config.yaml, output/data/runs/b30a-0209f019/202405090000/resources, output/data/runs/b30a-0209f019/202405090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405090000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405090000.log + jobid: 752 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411210000/config.yaml, output/data/runs/b30a-0209f019/202411210000/resources, output/data/runs/b30a-0209f019/202411210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411210000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411210000.log + jobid: 1340 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407150000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407150000.log + jobid: 198 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406060000/config.yaml, output/data/runs/b30a-0209f019/202406060000/resources, output/data/runs/b30a-0209f019/202406060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406060000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406060000.log + jobid: 836 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412190000/config.yaml, output/data/runs/b30a-0209f019/202412190000/resources, output/data/runs/b30a-0209f019/202412190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412190000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412190000.log + jobid: 1424 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401290000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401290000.log + jobid: 30 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403250000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403250000.log + jobid: 86 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411040000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411040000.log + jobid: 310 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407160000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407160000.log + jobid: 199 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401300000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401300000.log + jobid: 31 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403260000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403260000.log + jobid: 87 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411050000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411050000.log + jobid: 311 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403150000/config.yaml, output/data/runs/b30a-0209f019/202403150000/resources, output/data/runs/b30a-0209f019/202403150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403150000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403150000.log + jobid: 587 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405100000/config.yaml, output/data/runs/b30a-0209f019/202405100000/resources, output/data/runs/b30a-0209f019/202405100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405100000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405100000.log + jobid: 755 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411220000/config.yaml, output/data/runs/b30a-0209f019/202411220000/resources, output/data/runs/b30a-0209f019/202411220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411220000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411220000.log + jobid: 1343 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407170000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407170000.log + jobid: 200 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406070000/config.yaml, output/data/runs/b30a-0209f019/202406070000/resources, output/data/runs/b30a-0209f019/202406070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406070000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406070000.log + jobid: 839 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412200000/config.yaml, output/data/runs/b30a-0209f019/202412200000/resources, output/data/runs/b30a-0209f019/202412200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412200000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412200000.log + jobid: 1427 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202401310000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202401310000.log + jobid: 32 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202401310000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202401310000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403270000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403270000.log + jobid: 88 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411060000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411060000.log + jobid: 312 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407180000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407180000.log + jobid: 201 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402010000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402010000.log + jobid: 33 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403280000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403280000.log + jobid: 89 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411070000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411070000.log + jobid: 313 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403160000/config.yaml, output/data/runs/b30a-0209f019/202403160000/resources, output/data/runs/b30a-0209f019/202403160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403160000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403160000.log + jobid: 590 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405110000/config.yaml, output/data/runs/b30a-0209f019/202405110000/resources, output/data/runs/b30a-0209f019/202405110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405110000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405110000.log + jobid: 758 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411230000/config.yaml, output/data/runs/b30a-0209f019/202411230000/resources, output/data/runs/b30a-0209f019/202411230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411230000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411230000.log + jobid: 1346 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407190000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407190000.log + jobid: 202 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406080000/config.yaml, output/data/runs/b30a-0209f019/202406080000/resources, output/data/runs/b30a-0209f019/202406080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406080000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406080000.log + jobid: 842 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412210000/config.yaml, output/data/runs/b30a-0209f019/202412210000/resources, output/data/runs/b30a-0209f019/202412210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412210000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412210000.log + jobid: 1430 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402020000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402020000.log + jobid: 34 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403290000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403290000.log + jobid: 90 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411080000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411080000.log + jobid: 314 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407200000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407200000.log + jobid: 203 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402030000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402030000.log + jobid: 35 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403300000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403300000.log + jobid: 91 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411090000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411090000.log + jobid: 315 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410200000/config.yaml, output/data/runs/b30a-0209f019/202410200000/resources, output/data/runs/b30a-0209f019/202410200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410200000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410200000.log + jobid: 1244 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403170000/config.yaml, output/data/runs/b30a-0209f019/202403170000/resources, output/data/runs/b30a-0209f019/202403170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403170000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403170000.log + jobid: 593 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405120000/config.yaml, output/data/runs/b30a-0209f019/202405120000/resources, output/data/runs/b30a-0209f019/202405120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405120000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405120000.log + jobid: 761 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411240000/config.yaml, output/data/runs/b30a-0209f019/202411240000/resources, output/data/runs/b30a-0209f019/202411240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411240000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411240000.log + jobid: 1349 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407210000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407210000.log + jobid: 204 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406090000/config.yaml, output/data/runs/b30a-0209f019/202406090000/resources, output/data/runs/b30a-0209f019/202406090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406090000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406090000.log + jobid: 845 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412220000/config.yaml, output/data/runs/b30a-0209f019/202412220000/resources, output/data/runs/b30a-0209f019/202412220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412220000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412220000.log + jobid: 1433 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411100000/config.yaml, output/data/runs/b30a-0209f019/202411100000/resources, output/data/runs/b30a-0209f019/202411100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411100000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411100000.log + jobid: 1307 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402040000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402040000.log + jobid: 36 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403310000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403310000.log + jobid: 92 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403310000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403310000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411100000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411100000.log + jobid: 316 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407220000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407220000.log + jobid: 205 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402050000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402050000.log + jobid: 37 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404010000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404010000.log + jobid: 93 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411110000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411110000.log + jobid: 317 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403180000/config.yaml, output/data/runs/b30a-0209f019/202403180000/resources, output/data/runs/b30a-0209f019/202403180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403180000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403180000.log + jobid: 596 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405130000/config.yaml, output/data/runs/b30a-0209f019/202405130000/resources, output/data/runs/b30a-0209f019/202405130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405130000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405130000.log + jobid: 764 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411250000/config.yaml, output/data/runs/b30a-0209f019/202411250000/resources, output/data/runs/b30a-0209f019/202411250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411250000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411250000.log + jobid: 1352 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407230000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407230000.log + jobid: 206 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406100000/config.yaml, output/data/runs/b30a-0209f019/202406100000/resources, output/data/runs/b30a-0209f019/202406100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406100000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406100000.log + jobid: 848 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412230000/config.yaml, output/data/runs/b30a-0209f019/202412230000/resources, output/data/runs/b30a-0209f019/202412230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412230000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412230000.log + jobid: 1436 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402060000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402060000.log + jobid: 38 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404020000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404020000.log + jobid: 94 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410090000/config.yaml, output/data/runs/b30a-0209f019/202410090000/resources, output/data/runs/b30a-0209f019/202410090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410090000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410090000.log + jobid: 1211 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411120000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411120000.log + jobid: 318 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407240000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407240000.log + jobid: 207 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402070000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402070000.log + jobid: 39 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404030000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404030000.log + jobid: 95 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411130000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411130000.log + jobid: 319 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410250000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410250000.log + jobid: 300 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403190000/config.yaml, output/data/runs/b30a-0209f019/202403190000/resources, output/data/runs/b30a-0209f019/202403190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403190000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403190000.log + jobid: 599 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405140000/config.yaml, output/data/runs/b30a-0209f019/202405140000/resources, output/data/runs/b30a-0209f019/202405140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405140000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405140000.log + jobid: 767 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411260000/config.yaml, output/data/runs/b30a-0209f019/202411260000/resources, output/data/runs/b30a-0209f019/202411260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411260000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411260000.log + jobid: 1355 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407250000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407250000.log + jobid: 208 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406110000/config.yaml, output/data/runs/b30a-0209f019/202406110000/resources, output/data/runs/b30a-0209f019/202406110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406110000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406110000.log + jobid: 851 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412240000/config.yaml, output/data/runs/b30a-0209f019/202412240000/resources, output/data/runs/b30a-0209f019/202412240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412240000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412240000.log + jobid: 1439 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402080000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402080000.log + jobid: 40 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404040000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404040000.log + jobid: 96 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411140000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411140000.log + jobid: 320 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407260000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407260000.log + jobid: 209 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402090000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402090000.log + jobid: 41 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404050000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404050000.log + jobid: 97 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411150000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411150000.log + jobid: 321 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403200000/config.yaml, output/data/runs/b30a-0209f019/202403200000/resources, output/data/runs/b30a-0209f019/202403200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403200000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403200000.log + jobid: 602 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405150000/config.yaml, output/data/runs/b30a-0209f019/202405150000/resources, output/data/runs/b30a-0209f019/202405150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405150000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405150000.log + jobid: 770 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411270000/config.yaml, output/data/runs/b30a-0209f019/202411270000/resources, output/data/runs/b30a-0209f019/202411270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411270000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411270000.log + jobid: 1358 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406120000/config.yaml, output/data/runs/b30a-0209f019/202406120000/resources, output/data/runs/b30a-0209f019/202406120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406120000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406120000.log + jobid: 854 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407270000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407270000.log + jobid: 210 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412250000/config.yaml, output/data/runs/b30a-0209f019/202412250000/resources, output/data/runs/b30a-0209f019/202412250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412250000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412250000.log + jobid: 1442 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402100000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402100000.log + jobid: 42 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404060000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404060000.log + jobid: 98 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411160000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411160000.log + jobid: 322 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407280000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407280000.log + jobid: 211 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402110000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402110000.log + jobid: 43 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404070000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404070000.log + jobid: 99 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411170000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411170000.log + jobid: 323 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403210000/config.yaml, output/data/runs/b30a-0209f019/202403210000/resources, output/data/runs/b30a-0209f019/202403210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403210000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403210000.log + jobid: 605 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405160000/config.yaml, output/data/runs/b30a-0209f019/202405160000/resources, output/data/runs/b30a-0209f019/202405160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405160000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405160000.log + jobid: 773 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411280000/config.yaml, output/data/runs/b30a-0209f019/202411280000/resources, output/data/runs/b30a-0209f019/202411280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411280000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411280000.log + jobid: 1361 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407290000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407290000.log + jobid: 212 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406130000/config.yaml, output/data/runs/b30a-0209f019/202406130000/resources, output/data/runs/b30a-0209f019/202406130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406130000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406130000.log + jobid: 857 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402120000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402120000.log + jobid: 44 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404080000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404080000.log + jobid: 100 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411180000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411180000.log + jobid: 324 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407300000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407300000.log + jobid: 213 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402130000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402130000.log + jobid: 45 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404090000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404090000.log + jobid: 101 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411190000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411190000.log + jobid: 325 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410210000/config.yaml, output/data/runs/b30a-0209f019/202410210000/resources, output/data/runs/b30a-0209f019/202410210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410210000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410210000.log + jobid: 1247 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403220000/config.yaml, output/data/runs/b30a-0209f019/202403220000/resources, output/data/runs/b30a-0209f019/202403220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403220000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403220000.log + jobid: 608 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405170000/config.yaml, output/data/runs/b30a-0209f019/202405170000/resources, output/data/runs/b30a-0209f019/202405170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405170000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405170000.log + jobid: 776 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411290000/config.yaml, output/data/runs/b30a-0209f019/202411290000/resources, output/data/runs/b30a-0209f019/202411290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411290000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411290000.log + jobid: 1364 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202407310000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202407310000.log + jobid: 214 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202407310000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202407310000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406140000/config.yaml, output/data/runs/b30a-0209f019/202406140000/resources, output/data/runs/b30a-0209f019/202406140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406140000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406140000.log + jobid: 860 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411110000/config.yaml, output/data/runs/b30a-0209f019/202411110000/resources, output/data/runs/b30a-0209f019/202411110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411110000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411110000.log + jobid: 1310 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402140000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402140000.log + jobid: 46 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404100000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404100000.log + jobid: 102 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411200000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411200000.log + jobid: 326 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408010000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408010000.log + jobid: 215 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402150000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402150000.log + jobid: 47 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404110000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404110000.log + jobid: 103 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411210000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411210000.log + jobid: 327 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403230000/config.yaml, output/data/runs/b30a-0209f019/202403230000/resources, output/data/runs/b30a-0209f019/202403230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403230000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403230000.log + jobid: 611 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405180000/config.yaml, output/data/runs/b30a-0209f019/202405180000/resources, output/data/runs/b30a-0209f019/202405180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405180000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405180000.log + jobid: 779 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411300000/config.yaml, output/data/runs/b30a-0209f019/202411300000/resources, output/data/runs/b30a-0209f019/202411300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411300000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411300000.log + jobid: 1367 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408020000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408020000.log + jobid: 216 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406150000/config.yaml, output/data/runs/b30a-0209f019/202406150000/resources, output/data/runs/b30a-0209f019/202406150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406150000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406150000.log + jobid: 863 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402160000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402160000.log + jobid: 48 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404120000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404120000.log + jobid: 104 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411220000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411220000.log + jobid: 328 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410100000/config.yaml, output/data/runs/b30a-0209f019/202410100000/resources, output/data/runs/b30a-0209f019/202410100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410100000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410100000.log + jobid: 1214 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408030000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408030000.log + jobid: 217 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402170000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402170000.log + jobid: 49 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404130000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404130000.log + jobid: 105 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411230000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411230000.log + jobid: 329 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410270000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410270000.log + jobid: 302 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403240000/config.yaml, output/data/runs/b30a-0209f019/202403240000/resources, output/data/runs/b30a-0209f019/202403240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403240000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403240000.log + jobid: 614 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405190000/config.yaml, output/data/runs/b30a-0209f019/202405190000/resources, output/data/runs/b30a-0209f019/202405190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405190000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405190000.log + jobid: 782 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412010000/config.yaml, output/data/runs/b30a-0209f019/202412010000/resources, output/data/runs/b30a-0209f019/202412010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412010000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412010000.log + jobid: 1370 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408040000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408040000.log + jobid: 218 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406160000/config.yaml, output/data/runs/b30a-0209f019/202406160000/resources, output/data/runs/b30a-0209f019/202406160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406160000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406160000.log + jobid: 866 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402180000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402180000.log + jobid: 50 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404140000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404140000.log + jobid: 106 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411240000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411240000.log + jobid: 330 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408050000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408050000.log + jobid: 219 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402190000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402190000.log + jobid: 51 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404150000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404150000.log + jobid: 107 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411250000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411250000.log + jobid: 331 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403250000/config.yaml, output/data/runs/b30a-0209f019/202403250000/resources, output/data/runs/b30a-0209f019/202403250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403250000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403250000.log + jobid: 617 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405200000/config.yaml, output/data/runs/b30a-0209f019/202405200000/resources, output/data/runs/b30a-0209f019/202405200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405200000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405200000.log + jobid: 785 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412020000/config.yaml, output/data/runs/b30a-0209f019/202412020000/resources, output/data/runs/b30a-0209f019/202412020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412020000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412020000.log + jobid: 1373 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408060000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408060000.log + jobid: 220 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406170000/config.yaml, output/data/runs/b30a-0209f019/202406170000/resources, output/data/runs/b30a-0209f019/202406170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406170000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406170000.log + jobid: 869 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402200000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402200000.log + jobid: 52 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404160000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404160000.log + jobid: 108 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411260000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411260000.log + jobid: 332 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408070000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408070000.log + jobid: 221 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402210000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402210000.log + jobid: 53 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404170000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404170000.log + jobid: 109 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411270000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411270000.log + jobid: 333 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403260000/config.yaml, output/data/runs/b30a-0209f019/202403260000/resources, output/data/runs/b30a-0209f019/202403260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403260000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403260000.log + jobid: 620 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405210000/config.yaml, output/data/runs/b30a-0209f019/202405210000/resources, output/data/runs/b30a-0209f019/202405210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405210000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405210000.log + jobid: 788 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412030000/config.yaml, output/data/runs/b30a-0209f019/202412030000/resources, output/data/runs/b30a-0209f019/202412030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412030000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412030000.log + jobid: 1376 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408080000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408080000.log + jobid: 222 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406180000/config.yaml, output/data/runs/b30a-0209f019/202406180000/resources, output/data/runs/b30a-0209f019/202406180000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406180000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406180000.log + jobid: 872 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402220000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402220000.log + jobid: 54 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404180000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404180000.log + jobid: 110 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202401130000/config.yaml, output/data/runs/b30a-0209f019/202401130000/resources, output/data/runs/b30a-0209f019/202401130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202401130000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202401130000.log + jobid: 401 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202401130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411280000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411280000.log + jobid: 334 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408090000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408090000.log + jobid: 223 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407190000/config.yaml, output/data/runs/b30a-0209f019/202407190000/resources, output/data/runs/b30a-0209f019/202407190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407190000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407190000.log + jobid: 965 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402230000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402230000.log + jobid: 55 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404190000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404190000.log + jobid: 111 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411290000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411290000.log + jobid: 335 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410220000/config.yaml, output/data/runs/b30a-0209f019/202410220000/resources, output/data/runs/b30a-0209f019/202410220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410220000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410220000.log + jobid: 1250 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405220000/config.yaml, output/data/runs/b30a-0209f019/202405220000/resources, output/data/runs/b30a-0209f019/202405220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405220000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405220000.log + jobid: 791 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403270000/config.yaml, output/data/runs/b30a-0209f019/202403270000/resources, output/data/runs/b30a-0209f019/202403270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403270000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403270000.log + jobid: 623 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412040000/config.yaml, output/data/runs/b30a-0209f019/202412040000/resources, output/data/runs/b30a-0209f019/202412040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412040000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412040000.log + jobid: 1379 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408100000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408100000.log + jobid: 224 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406190000/config.yaml, output/data/runs/b30a-0209f019/202406190000/resources, output/data/runs/b30a-0209f019/202406190000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406190000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406190000.log + jobid: 875 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411120000/config.yaml, output/data/runs/b30a-0209f019/202411120000/resources, output/data/runs/b30a-0209f019/202411120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411120000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411120000.log + jobid: 1313 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402240000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402240000.log + jobid: 56 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202411300000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202411300000.log + jobid: 336 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202411300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202411300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404200000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404200000.log + jobid: 112 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408110000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408110000.log + jobid: 225 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402250000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402250000.log + jobid: 57 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404210000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404210000.log + jobid: 113 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412010000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412010000.log + jobid: 337 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403280000/config.yaml, output/data/runs/b30a-0209f019/202403280000/resources, output/data/runs/b30a-0209f019/202403280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403280000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403280000.log + jobid: 626 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405230000/config.yaml, output/data/runs/b30a-0209f019/202405230000/resources, output/data/runs/b30a-0209f019/202405230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405230000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405230000.log + jobid: 794 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412050000/config.yaml, output/data/runs/b30a-0209f019/202412050000/resources, output/data/runs/b30a-0209f019/202412050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412050000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412050000.log + jobid: 1382 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408120000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408120000.log + jobid: 226 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406200000/config.yaml, output/data/runs/b30a-0209f019/202406200000/resources, output/data/runs/b30a-0209f019/202406200000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406200000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406200000.log + jobid: 878 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402260000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402260000.log + jobid: 58 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404220000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404220000.log + jobid: 114 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412020000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412020000.log + jobid: 338 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410110000/config.yaml, output/data/runs/b30a-0209f019/202410110000/resources, output/data/runs/b30a-0209f019/202410110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410110000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410110000.log + jobid: 1217 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408130000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408130000.log + jobid: 227 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402270000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402270000.log + jobid: 59 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404230000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404230000.log + jobid: 115 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412030000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412030000.log + jobid: 339 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410290000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410290000.log + jobid: 304 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403290000/config.yaml, output/data/runs/b30a-0209f019/202403290000/resources, output/data/runs/b30a-0209f019/202403290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403290000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403290000.log + jobid: 629 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405240000/config.yaml, output/data/runs/b30a-0209f019/202405240000/resources, output/data/runs/b30a-0209f019/202405240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405240000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405240000.log + jobid: 797 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412060000/config.yaml, output/data/runs/b30a-0209f019/202412060000/resources, output/data/runs/b30a-0209f019/202412060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412060000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412060000.log + jobid: 1385 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408140000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408140000.log + jobid: 228 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406210000/config.yaml, output/data/runs/b30a-0209f019/202406210000/resources, output/data/runs/b30a-0209f019/202406210000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406210000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406210000.log + jobid: 881 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402280000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402280000.log + jobid: 60 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404240000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404240000.log + jobid: 116 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412040000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412040000.log + jobid: 340 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408150000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408150000.log + jobid: 229 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202402290000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202402290000.log + jobid: 61 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202402290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202402290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404250000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404250000.log + jobid: 117 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412050000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412050000.log + jobid: 341 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403300000/config.yaml, output/data/runs/b30a-0209f019/202403300000/resources, output/data/runs/b30a-0209f019/202403300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403300000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403300000.log + jobid: 632 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405250000/config.yaml, output/data/runs/b30a-0209f019/202405250000/resources, output/data/runs/b30a-0209f019/202405250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405250000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405250000.log + jobid: 800 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412070000/config.yaml, output/data/runs/b30a-0209f019/202412070000/resources, output/data/runs/b30a-0209f019/202412070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412070000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412070000.log + jobid: 1388 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408160000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408160000.log + jobid: 230 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406220000/config.yaml, output/data/runs/b30a-0209f019/202406220000/resources, output/data/runs/b30a-0209f019/202406220000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406220000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406220000.log + jobid: 884 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403010000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403010000.log + jobid: 62 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404260000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404260000.log + jobid: 118 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412060000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412060000.log + jobid: 342 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408170000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408170000.log + jobid: 231 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403020000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403020000.log + jobid: 63 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404270000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404270000.log + jobid: 119 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412070000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412070000.log + jobid: 343 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202403310000/config.yaml, output/data/runs/b30a-0209f019/202403310000/resources, output/data/runs/b30a-0209f019/202403310000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202403310000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202403310000.log + jobid: 635 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202403310000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405260000/config.yaml, output/data/runs/b30a-0209f019/202405260000/resources, output/data/runs/b30a-0209f019/202405260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405260000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405260000.log + jobid: 803 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412080000/config.yaml, output/data/runs/b30a-0209f019/202412080000/resources, output/data/runs/b30a-0209f019/202412080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412080000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412080000.log + jobid: 1391 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408180000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408180000.log + jobid: 232 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406230000/config.yaml, output/data/runs/b30a-0209f019/202406230000/resources, output/data/runs/b30a-0209f019/202406230000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406230000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406230000.log + jobid: 887 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403030000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403030000.log + jobid: 64 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404280000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404280000.log + jobid: 120 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412080000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412080000.log + jobid: 344 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408190000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408190000.log + jobid: 233 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403040000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403040000.log + jobid: 65 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404290000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404290000.log + jobid: 121 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412090000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412090000.log + jobid: 345 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404010000/config.yaml, output/data/runs/b30a-0209f019/202404010000/resources, output/data/runs/b30a-0209f019/202404010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404010000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404010000.log + jobid: 638 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405270000/config.yaml, output/data/runs/b30a-0209f019/202405270000/resources, output/data/runs/b30a-0209f019/202405270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405270000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405270000.log + jobid: 806 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412090000/config.yaml, output/data/runs/b30a-0209f019/202412090000/resources, output/data/runs/b30a-0209f019/202412090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412090000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412090000.log + jobid: 1394 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408200000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408200000.log + jobid: 234 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406240000/config.yaml, output/data/runs/b30a-0209f019/202406240000/resources, output/data/runs/b30a-0209f019/202406240000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406240000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406240000.log + jobid: 890 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411130000/config.yaml, output/data/runs/b30a-0209f019/202411130000/resources, output/data/runs/b30a-0209f019/202411130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411130000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411130000.log + jobid: 1316 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403050000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403050000.log + jobid: 66 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202404300000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202404300000.log + jobid: 122 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202404300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202404300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412100000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412100000.log + jobid: 346 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408210000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408210000.log + jobid: 235 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403060000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403060000.log + jobid: 67 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405010000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405010000.log + jobid: 123 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412110000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412110000.log + jobid: 347 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404020000/config.yaml, output/data/runs/b30a-0209f019/202404020000/resources, output/data/runs/b30a-0209f019/202404020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404020000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404020000.log + jobid: 641 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405280000/config.yaml, output/data/runs/b30a-0209f019/202405280000/resources, output/data/runs/b30a-0209f019/202405280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405280000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405280000.log + jobid: 809 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412100000/config.yaml, output/data/runs/b30a-0209f019/202412100000/resources, output/data/runs/b30a-0209f019/202412100000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412100000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412100000.log + jobid: 1397 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408220000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408220000.log + jobid: 236 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406250000/config.yaml, output/data/runs/b30a-0209f019/202406250000/resources, output/data/runs/b30a-0209f019/202406250000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406250000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406250000.log + jobid: 893 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403070000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403070000.log + jobid: 68 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405020000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405020000.log + jobid: 124 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412120000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412120000.log + jobid: 348 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202410120000/config.yaml, output/data/runs/b30a-0209f019/202410120000/resources, output/data/runs/b30a-0209f019/202410120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202410120000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202410120000.log + jobid: 1220 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202410120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408230000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408230000.log + jobid: 237 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403080000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403080000.log + jobid: 69 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405030000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405030000.log + jobid: 125 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412130000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412130000.log + jobid: 349 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202410310000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202410310000.log + jobid: 306 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202410310000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202410310000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404030000/config.yaml, output/data/runs/b30a-0209f019/202404030000/resources, output/data/runs/b30a-0209f019/202404030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404030000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404030000.log + jobid: 644 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405290000/config.yaml, output/data/runs/b30a-0209f019/202405290000/resources, output/data/runs/b30a-0209f019/202405290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405290000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405290000.log + jobid: 812 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412110000/config.yaml, output/data/runs/b30a-0209f019/202412110000/resources, output/data/runs/b30a-0209f019/202412110000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412110000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412110000.log + jobid: 1400 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408240000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408240000.log + jobid: 238 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406260000/config.yaml, output/data/runs/b30a-0209f019/202406260000/resources, output/data/runs/b30a-0209f019/202406260000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406260000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406260000.log + jobid: 896 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403090000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403090000.log + jobid: 70 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405040000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405040000.log + jobid: 126 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412140000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412140000.log + jobid: 350 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408250000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408250000.log + jobid: 239 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403100000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403100000.log + jobid: 71 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405050000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405050000.log + jobid: 127 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412150000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412150000.log + jobid: 351 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404040000/config.yaml, output/data/runs/b30a-0209f019/202404040000/resources, output/data/runs/b30a-0209f019/202404040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404040000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404040000.log + jobid: 647 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405300000/config.yaml, output/data/runs/b30a-0209f019/202405300000/resources, output/data/runs/b30a-0209f019/202405300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405300000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405300000.log + jobid: 815 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412120000/config.yaml, output/data/runs/b30a-0209f019/202412120000/resources, output/data/runs/b30a-0209f019/202412120000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412120000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412120000.log + jobid: 1403 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408260000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408260000.log + jobid: 240 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408260000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406270000/config.yaml, output/data/runs/b30a-0209f019/202406270000/resources, output/data/runs/b30a-0209f019/202406270000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406270000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406270000.log + jobid: 899 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403110000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403110000.log + jobid: 72 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405060000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405060000.log + jobid: 128 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412160000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412160000.log + jobid: 352 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408270000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408270000.log + jobid: 241 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408270000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403120000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403120000.log + jobid: 73 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405070000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405070000.log + jobid: 129 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405070000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412170000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412170000.log + jobid: 353 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404050000/config.yaml, output/data/runs/b30a-0209f019/202404050000/resources, output/data/runs/b30a-0209f019/202404050000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404050000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404050000.log + jobid: 650 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202405310000/config.yaml, output/data/runs/b30a-0209f019/202405310000/resources, output/data/runs/b30a-0209f019/202405310000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202405310000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202405310000.log + jobid: 818 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202405310000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412130000/config.yaml, output/data/runs/b30a-0209f019/202412130000/resources, output/data/runs/b30a-0209f019/202412130000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412130000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412130000.log + jobid: 1406 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408280000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408280000.log + jobid: 242 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408280000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406280000/config.yaml, output/data/runs/b30a-0209f019/202406280000/resources, output/data/runs/b30a-0209f019/202406280000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406280000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406280000.log + jobid: 902 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403130000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403130000.log + jobid: 74 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405080000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405080000.log + jobid: 130 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405080000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412180000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412180000.log + jobid: 354 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408290000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408290000.log + jobid: 243 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408290000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403140000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403140000.log + jobid: 75 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405090000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405090000.log + jobid: 131 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405090000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412190000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412190000.log + jobid: 355 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404060000/config.yaml, output/data/runs/b30a-0209f019/202404060000/resources, output/data/runs/b30a-0209f019/202404060000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404060000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404060000.log + jobid: 653 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406010000/config.yaml, output/data/runs/b30a-0209f019/202406010000/resources, output/data/runs/b30a-0209f019/202406010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406010000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406010000.log + jobid: 821 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412140000/config.yaml, output/data/runs/b30a-0209f019/202412140000/resources, output/data/runs/b30a-0209f019/202412140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412140000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412140000.log + jobid: 1409 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408300000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408300000.log + jobid: 244 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408300000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406290000/config.yaml, output/data/runs/b30a-0209f019/202406290000/resources, output/data/runs/b30a-0209f019/202406290000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406290000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406290000.log + jobid: 905 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411140000/config.yaml, output/data/runs/b30a-0209f019/202411140000/resources, output/data/runs/b30a-0209f019/202411140000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411140000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411140000.log + jobid: 1319 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403150000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403150000.log + jobid: 76 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405100000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405100000.log + jobid: 132 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405100000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412200000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412200000.log + jobid: 356 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202408310000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202408310000.log + jobid: 245 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202408310000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202408310000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403160000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403160000.log + jobid: 77 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405110000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405110000.log + jobid: 133 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405110000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412210000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412210000.log + jobid: 357 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404070000/config.yaml, output/data/runs/b30a-0209f019/202404070000/resources, output/data/runs/b30a-0209f019/202404070000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404070000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404070000.log + jobid: 656 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406020000/config.yaml, output/data/runs/b30a-0209f019/202406020000/resources, output/data/runs/b30a-0209f019/202406020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406020000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406020000.log + jobid: 824 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412150000/config.yaml, output/data/runs/b30a-0209f019/202412150000/resources, output/data/runs/b30a-0209f019/202412150000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412150000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412150000.log + jobid: 1412 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409010000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409010000.log + jobid: 246 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409010000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406300000/config.yaml, output/data/runs/b30a-0209f019/202406300000/resources, output/data/runs/b30a-0209f019/202406300000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406300000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406300000.log + jobid: 908 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403170000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403170000.log + jobid: 78 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405120000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405120000.log + jobid: 134 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405120000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412220000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412220000.log + jobid: 358 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409020000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409020000.log + jobid: 247 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409020000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403180000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403180000.log + jobid: 79 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403180000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405130000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405130000.log + jobid: 135 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405130000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412230000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412230000.log + jobid: 359 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412230000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202411030000/config.yaml, output/data/runs/b30a-0209f019/202411030000/resources, output/data/runs/b30a-0209f019/202411030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202411030000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202411030000.log + jobid: 1286 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202411030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404080000/config.yaml, output/data/runs/b30a-0209f019/202404080000/resources, output/data/runs/b30a-0209f019/202404080000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404080000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404080000.log + jobid: 659 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406030000/config.yaml, output/data/runs/b30a-0209f019/202406030000/resources, output/data/runs/b30a-0209f019/202406030000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406030000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406030000.log + jobid: 827 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412160000/config.yaml, output/data/runs/b30a-0209f019/202412160000/resources, output/data/runs/b30a-0209f019/202412160000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412160000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412160000.log + jobid: 1415 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409030000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409030000.log + jobid: 248 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409030000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407010000/config.yaml, output/data/runs/b30a-0209f019/202407010000/resources, output/data/runs/b30a-0209f019/202407010000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407010000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407010000.log + jobid: 911 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403190000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403190000.log + jobid: 80 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403190000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405140000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405140000.log + jobid: 136 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405140000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412240000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412240000.log + jobid: 360 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412240000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409040000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409040000.log + jobid: 249 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409040000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403200000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403200000.log + jobid: 81 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403200000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405150000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405150000.log + jobid: 137 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405150000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202412250000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202412250000.log + jobid: 361 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202412250000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202412250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202404090000/config.yaml, output/data/runs/b30a-0209f019/202404090000/resources, output/data/runs/b30a-0209f019/202404090000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202404090000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202404090000.log + jobid: 662 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202404090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202406040000/config.yaml, output/data/runs/b30a-0209f019/202406040000/resources, output/data/runs/b30a-0209f019/202406040000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202406040000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202406040000.log + jobid: 830 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202406040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202412170000/config.yaml, output/data/runs/b30a-0209f019/202412170000/resources, output/data/runs/b30a-0209f019/202412170000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202412170000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202412170000.log + jobid: 1418 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202412170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409050000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409050000.log + jobid: 250 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409050000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule prepare_inference_forecaster: + input: output/data/runs/b30a-0209f019/pyproject.toml, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/resources/inference/configs/sgm-forecaster-global-ich1.yaml + output: output/data/runs/b30a-0209f019/202407020000/config.yaml, output/data/runs/b30a-0209f019/202407020000/resources, output/data/runs/b30a-0209f019/202407020000/grib, output/logs/prepare_inference_forecaster/b30a-0209f019-202407020000.ok + log: output/logs/prepare_inference_forecaster/b30a-0209f019-202407020000.log + jobid: 914 + reason: Code has changed since last execution + wildcards: run_id=b30a-0209f019, init_time=202407020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403210000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403210000.log + jobid: 82 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403210000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405160000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405160000.log + jobid: 138 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405160000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202409060000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202409060000.log + jobid: 251 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202409060000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202409060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202403220000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202403220000.log + jobid: 83 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202403220000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202403220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_baseline: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, /store_new/mch/msopr/ml/ICON-CH1-EPS/FCST24.zarr, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/baselines/ICON-CH1-EPS/202405170000/verif.nc + log: output/logs/verif_metrics_baseline/ICON-CH1-EPS-202405170000.log + jobid: 139 + reason: Missing output files: output/data/baselines/ICON-CH1-EPS/202405170000/verif.nc; Updated input files: workflow/scripts/verif_single_init.py + wildcards: baseline_id=ICON-CH1-EPS, init_time=202405170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +Would remove temporary output output/data/runs/b30a-0209f019/.venv + +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401190000.ok + log: output/logs/execute_inference/b30a-0209f019-202401190000.log + jobid: 418 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202401190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408020000.ok + log: output/logs/execute_inference/b30a-0209f019-202408020000.log + jobid: 1006 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408020000.ok + wildcards: run_id=b30a-0209f019, init_time=202408020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407050000.ok + log: output/logs/execute_inference/b30a-0209f019-202407050000.log + jobid: 922 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407050000.ok + wildcards: run_id=b30a-0209f019, init_time=202407050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404120000.ok + log: output/logs/execute_inference/b30a-0209f019-202404120000.log + jobid: 670 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404120000.ok + wildcards: run_id=b30a-0209f019, init_time=202404120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402160000.ok + log: output/logs/execute_inference/b30a-0209f019-202402160000.log + jobid: 502 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402160000.ok + wildcards: run_id=b30a-0209f019, init_time=202402160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408300000.ok + log: output/logs/execute_inference/b30a-0209f019-202408300000.log + jobid: 1090 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202408300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409270000.ok + log: output/logs/execute_inference/b30a-0209f019-202409270000.log + jobid: 1174 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409270000.ok + wildcards: run_id=b30a-0209f019, init_time=202409270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410250000.ok + log: output/logs/execute_inference/b30a-0209f019-202410250000.log + jobid: 1258 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202410250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408310000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408310000.ok + log: output/logs/execute_inference/b30a-0209f019-202408310000.log + jobid: 1093 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408310000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202408310000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402170000.ok + log: output/logs/execute_inference/b30a-0209f019-202402170000.log + jobid: 505 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202402170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401200000.ok + log: output/logs/execute_inference/b30a-0209f019-202401200000.log + jobid: 421 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401200000.ok + wildcards: run_id=b30a-0209f019, init_time=202401200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404130000.ok + log: output/logs/execute_inference/b30a-0209f019-202404130000.log + jobid: 673 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404130000.ok + wildcards: run_id=b30a-0209f019, init_time=202404130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407060000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407060000.ok + log: output/logs/execute_inference/b30a-0209f019-202407060000.log + jobid: 925 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407060000.ok + wildcards: run_id=b30a-0209f019, init_time=202407060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408030000.ok + log: output/logs/execute_inference/b30a-0209f019-202408030000.log + jobid: 1009 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408030000.ok + wildcards: run_id=b30a-0209f019, init_time=202408030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409280000.ok + log: output/logs/execute_inference/b30a-0209f019-202409280000.log + jobid: 1177 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202409280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410260000.ok + log: output/logs/execute_inference/b30a-0209f019-202410260000.log + jobid: 1261 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410260000.ok + wildcards: run_id=b30a-0209f019, init_time=202410260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404140000.ok + log: output/logs/execute_inference/b30a-0209f019-202404140000.log + jobid: 676 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404140000.ok + wildcards: run_id=b30a-0209f019, init_time=202404140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401210000.ok + log: output/logs/execute_inference/b30a-0209f019-202401210000.log + jobid: 424 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401210000.ok + wildcards: run_id=b30a-0209f019, init_time=202401210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402180000.ok + log: output/logs/execute_inference/b30a-0209f019-202402180000.log + jobid: 508 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402180000.ok + wildcards: run_id=b30a-0209f019, init_time=202402180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407070000.ok + log: output/logs/execute_inference/b30a-0209f019-202407070000.log + jobid: 928 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202407070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408040000.ok + log: output/logs/execute_inference/b30a-0209f019-202408040000.log + jobid: 1012 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408040000.ok + wildcards: run_id=b30a-0209f019, init_time=202408040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409010000.ok + log: output/logs/execute_inference/b30a-0209f019-202409010000.log + jobid: 1096 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409010000.ok + wildcards: run_id=b30a-0209f019, init_time=202409010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409290000.ok + log: output/logs/execute_inference/b30a-0209f019-202409290000.log + jobid: 1180 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202409290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410270000.ok + log: output/logs/execute_inference/b30a-0209f019-202410270000.log + jobid: 1264 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410270000.ok + wildcards: run_id=b30a-0209f019, init_time=202410270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401220000.ok + log: output/logs/execute_inference/b30a-0209f019-202401220000.log + jobid: 427 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401220000.ok + wildcards: run_id=b30a-0209f019, init_time=202401220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402190000.ok + log: output/logs/execute_inference/b30a-0209f019-202402190000.log + jobid: 511 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402190000.ok + wildcards: run_id=b30a-0209f019, init_time=202402190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404150000.ok + log: output/logs/execute_inference/b30a-0209f019-202404150000.log + jobid: 679 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202404150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407080000.ok + log: output/logs/execute_inference/b30a-0209f019-202407080000.log + jobid: 931 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407080000.ok + wildcards: run_id=b30a-0209f019, init_time=202407080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408050000.ok + log: output/logs/execute_inference/b30a-0209f019-202408050000.log + jobid: 1015 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408050000.ok + wildcards: run_id=b30a-0209f019, init_time=202408050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409020000.ok + log: output/logs/execute_inference/b30a-0209f019-202409020000.log + jobid: 1099 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409020000.ok + wildcards: run_id=b30a-0209f019, init_time=202409020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409300000.ok + log: output/logs/execute_inference/b30a-0209f019-202409300000.log + jobid: 1183 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409300000.ok + wildcards: run_id=b30a-0209f019, init_time=202409300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410280000.ok + log: output/logs/execute_inference/b30a-0209f019-202410280000.log + jobid: 1267 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410280000.ok + wildcards: run_id=b30a-0209f019, init_time=202410280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401230000.ok + log: output/logs/execute_inference/b30a-0209f019-202401230000.log + jobid: 430 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202401230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402200000.ok + log: output/logs/execute_inference/b30a-0209f019-202402200000.log + jobid: 514 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402200000.ok + wildcards: run_id=b30a-0209f019, init_time=202402200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404160000.ok + log: output/logs/execute_inference/b30a-0209f019-202404160000.log + jobid: 682 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404160000.ok + wildcards: run_id=b30a-0209f019, init_time=202404160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407090000.ok + log: output/logs/execute_inference/b30a-0209f019-202407090000.log + jobid: 934 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407090000.ok + wildcards: run_id=b30a-0209f019, init_time=202407090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408060000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408060000.ok + log: output/logs/execute_inference/b30a-0209f019-202408060000.log + jobid: 1018 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408060000.ok + wildcards: run_id=b30a-0209f019, init_time=202408060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409030000.ok + log: output/logs/execute_inference/b30a-0209f019-202409030000.log + jobid: 1102 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202409030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410010000.ok + log: output/logs/execute_inference/b30a-0209f019-202410010000.log + jobid: 1186 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410010000.ok + wildcards: run_id=b30a-0209f019, init_time=202410010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410290000.ok + log: output/logs/execute_inference/b30a-0209f019-202410290000.log + jobid: 1270 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202410290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401240000.ok + log: output/logs/execute_inference/b30a-0209f019-202401240000.log + jobid: 433 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202401240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402210000.ok + log: output/logs/execute_inference/b30a-0209f019-202402210000.log + jobid: 517 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202402210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404170000.ok + log: output/logs/execute_inference/b30a-0209f019-202404170000.log + jobid: 685 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404170000.ok + wildcards: run_id=b30a-0209f019, init_time=202404170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407100000.ok + log: output/logs/execute_inference/b30a-0209f019-202407100000.log + jobid: 937 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407100000.ok + wildcards: run_id=b30a-0209f019, init_time=202407100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408070000.ok + log: output/logs/execute_inference/b30a-0209f019-202408070000.log + jobid: 1021 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202408070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409040000.ok + log: output/logs/execute_inference/b30a-0209f019-202409040000.log + jobid: 1105 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202409040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410020000.ok + log: output/logs/execute_inference/b30a-0209f019-202410020000.log + jobid: 1189 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410020000.ok + wildcards: run_id=b30a-0209f019, init_time=202410020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410300000.ok + log: output/logs/execute_inference/b30a-0209f019-202410300000.log + jobid: 1273 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202410300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401250000.ok + log: output/logs/execute_inference/b30a-0209f019-202401250000.log + jobid: 436 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202401250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402220000.ok + log: output/logs/execute_inference/b30a-0209f019-202402220000.log + jobid: 520 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402220000.ok + wildcards: run_id=b30a-0209f019, init_time=202402220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404180000.ok + log: output/logs/execute_inference/b30a-0209f019-202404180000.log + jobid: 688 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202404180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407110000.ok + log: output/logs/execute_inference/b30a-0209f019-202407110000.log + jobid: 940 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407110000.ok + wildcards: run_id=b30a-0209f019, init_time=202407110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408080000.ok + log: output/logs/execute_inference/b30a-0209f019-202408080000.log + jobid: 1024 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408080000.ok + wildcards: run_id=b30a-0209f019, init_time=202408080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409050000.ok + log: output/logs/execute_inference/b30a-0209f019-202409050000.log + jobid: 1108 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409050000.ok + wildcards: run_id=b30a-0209f019, init_time=202409050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410030000.ok + log: output/logs/execute_inference/b30a-0209f019-202410030000.log + jobid: 1192 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202410030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410310000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410310000.ok + log: output/logs/execute_inference/b30a-0209f019-202410310000.log + jobid: 1276 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410310000.ok + wildcards: run_id=b30a-0209f019, init_time=202410310000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401260000.ok + log: output/logs/execute_inference/b30a-0209f019-202401260000.log + jobid: 439 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401260000.ok + wildcards: run_id=b30a-0209f019, init_time=202401260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402230000.ok + log: output/logs/execute_inference/b30a-0209f019-202402230000.log + jobid: 523 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402230000.ok + wildcards: run_id=b30a-0209f019, init_time=202402230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404190000.ok + log: output/logs/execute_inference/b30a-0209f019-202404190000.log + jobid: 691 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202404190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407120000.ok + log: output/logs/execute_inference/b30a-0209f019-202407120000.log + jobid: 943 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202407120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410040000.ok + log: output/logs/execute_inference/b30a-0209f019-202410040000.log + jobid: 1195 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410040000.ok + wildcards: run_id=b30a-0209f019, init_time=202410040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411010000.ok + log: output/logs/execute_inference/b30a-0209f019-202411010000.log + jobid: 1279 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411010000.ok + wildcards: run_id=b30a-0209f019, init_time=202411010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408090000.ok + log: output/logs/execute_inference/b30a-0209f019-202408090000.log + jobid: 1027 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408090000.ok + wildcards: run_id=b30a-0209f019, init_time=202408090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409060000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409060000.ok + log: output/logs/execute_inference/b30a-0209f019-202409060000.log + jobid: 1111 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409060000.ok + wildcards: run_id=b30a-0209f019, init_time=202409060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404200000.ok + log: output/logs/execute_inference/b30a-0209f019-202404200000.log + jobid: 694 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404200000.ok + wildcards: run_id=b30a-0209f019, init_time=202404200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407130000.ok + log: output/logs/execute_inference/b30a-0209f019-202407130000.log + jobid: 946 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202407130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408100000.ok + log: output/logs/execute_inference/b30a-0209f019-202408100000.log + jobid: 1030 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408100000.ok + wildcards: run_id=b30a-0209f019, init_time=202408100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409070000.ok + log: output/logs/execute_inference/b30a-0209f019-202409070000.log + jobid: 1114 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202409070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401270000.ok + log: output/logs/execute_inference/b30a-0209f019-202401270000.log + jobid: 442 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401270000.ok + wildcards: run_id=b30a-0209f019, init_time=202401270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402240000.ok + log: output/logs/execute_inference/b30a-0209f019-202402240000.log + jobid: 526 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402240000.ok + wildcards: run_id=b30a-0209f019, init_time=202402240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410050000.ok + log: output/logs/execute_inference/b30a-0209f019-202410050000.log + jobid: 1198 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410050000.ok + wildcards: run_id=b30a-0209f019, init_time=202410050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411020000.ok + log: output/logs/execute_inference/b30a-0209f019-202411020000.log + jobid: 1282 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411020000.ok + wildcards: run_id=b30a-0209f019, init_time=202411020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402250000.ok + log: output/logs/execute_inference/b30a-0209f019-202402250000.log + jobid: 529 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202402250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401280000.ok + log: output/logs/execute_inference/b30a-0209f019-202401280000.log + jobid: 445 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401280000.ok + wildcards: run_id=b30a-0209f019, init_time=202401280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404210000.ok + log: output/logs/execute_inference/b30a-0209f019-202404210000.log + jobid: 697 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404210000.ok + wildcards: run_id=b30a-0209f019, init_time=202404210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408110000.ok + log: output/logs/execute_inference/b30a-0209f019-202408110000.log + jobid: 1033 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408110000.ok + wildcards: run_id=b30a-0209f019, init_time=202408110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409080000.ok + log: output/logs/execute_inference/b30a-0209f019-202409080000.log + jobid: 1117 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202409080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407140000.ok + log: output/logs/execute_inference/b30a-0209f019-202407140000.log + jobid: 949 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407140000.ok + wildcards: run_id=b30a-0209f019, init_time=202407140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410060000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410060000.ok + log: output/logs/execute_inference/b30a-0209f019-202410060000.log + jobid: 1201 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410060000.ok + wildcards: run_id=b30a-0209f019, init_time=202410060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411030000.ok + log: output/logs/execute_inference/b30a-0209f019-202411030000.log + jobid: 1285 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202411030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408120000.ok + log: output/logs/execute_inference/b30a-0209f019-202408120000.log + jobid: 1036 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408120000.ok + wildcards: run_id=b30a-0209f019, init_time=202408120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401290000.ok + log: output/logs/execute_inference/b30a-0209f019-202401290000.log + jobid: 448 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202401290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409090000.ok + log: output/logs/execute_inference/b30a-0209f019-202409090000.log + jobid: 1120 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409090000.ok + wildcards: run_id=b30a-0209f019, init_time=202409090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402260000.ok + log: output/logs/execute_inference/b30a-0209f019-202402260000.log + jobid: 532 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202402260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404220000.ok + log: output/logs/execute_inference/b30a-0209f019-202404220000.log + jobid: 700 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404220000.ok + wildcards: run_id=b30a-0209f019, init_time=202404220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407150000.ok + log: output/logs/execute_inference/b30a-0209f019-202407150000.log + jobid: 952 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407150000.ok + wildcards: run_id=b30a-0209f019, init_time=202407150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410070000.ok + log: output/logs/execute_inference/b30a-0209f019-202410070000.log + jobid: 1204 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410070000.ok + wildcards: run_id=b30a-0209f019, init_time=202410070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411040000.ok + log: output/logs/execute_inference/b30a-0209f019-202411040000.log + jobid: 1288 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411040000.ok + wildcards: run_id=b30a-0209f019, init_time=202411040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408130000.ok + log: output/logs/execute_inference/b30a-0209f019-202408130000.log + jobid: 1039 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202408130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401300000.ok + log: output/logs/execute_inference/b30a-0209f019-202401300000.log + jobid: 451 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202401300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409100000.ok + log: output/logs/execute_inference/b30a-0209f019-202409100000.log + jobid: 1123 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409100000.ok + wildcards: run_id=b30a-0209f019, init_time=202409100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402270000.ok + log: output/logs/execute_inference/b30a-0209f019-202402270000.log + jobid: 535 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402270000.ok + wildcards: run_id=b30a-0209f019, init_time=202402270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404230000.ok + log: output/logs/execute_inference/b30a-0209f019-202404230000.log + jobid: 703 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404230000.ok + wildcards: run_id=b30a-0209f019, init_time=202404230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407160000.ok + log: output/logs/execute_inference/b30a-0209f019-202407160000.log + jobid: 955 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407160000.ok + wildcards: run_id=b30a-0209f019, init_time=202407160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410080000.ok + log: output/logs/execute_inference/b30a-0209f019-202410080000.log + jobid: 1207 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202410080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411050000.ok + log: output/logs/execute_inference/b30a-0209f019-202411050000.log + jobid: 1291 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411050000.ok + wildcards: run_id=b30a-0209f019, init_time=202411050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402280000.ok + log: output/logs/execute_inference/b30a-0209f019-202402280000.log + jobid: 538 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402280000.ok + wildcards: run_id=b30a-0209f019, init_time=202402280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404240000.ok + log: output/logs/execute_inference/b30a-0209f019-202404240000.log + jobid: 706 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202404240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407170000.ok + log: output/logs/execute_inference/b30a-0209f019-202407170000.log + jobid: 958 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202407170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401310000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401310000.ok + log: output/logs/execute_inference/b30a-0209f019-202401310000.log + jobid: 454 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401310000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202401310000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409110000.ok + log: output/logs/execute_inference/b30a-0209f019-202409110000.log + jobid: 1126 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409110000.ok + wildcards: run_id=b30a-0209f019, init_time=202409110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411060000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411060000.ok + log: output/logs/execute_inference/b30a-0209f019-202411060000.log + jobid: 1294 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411060000.ok + wildcards: run_id=b30a-0209f019, init_time=202411060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408140000.ok + log: output/logs/execute_inference/b30a-0209f019-202408140000.log + jobid: 1042 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408140000.ok + wildcards: run_id=b30a-0209f019, init_time=202408140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410090000.ok + log: output/logs/execute_inference/b30a-0209f019-202410090000.log + jobid: 1210 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410090000.ok + wildcards: run_id=b30a-0209f019, init_time=202410090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401030000.ok + log: output/logs/execute_inference/b30a-0209f019-202401030000.log + jobid: 370 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202401030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402010000.ok + log: output/logs/execute_inference/b30a-0209f019-202402010000.log + jobid: 457 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402010000.ok + wildcards: run_id=b30a-0209f019, init_time=202402010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408150000.ok + log: output/logs/execute_inference/b30a-0209f019-202408150000.log + jobid: 1045 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408150000.ok + wildcards: run_id=b30a-0209f019, init_time=202408150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409120000.ok + log: output/logs/execute_inference/b30a-0209f019-202409120000.log + jobid: 1129 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409120000.ok + wildcards: run_id=b30a-0209f019, init_time=202409120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402290000.ok + log: output/logs/execute_inference/b30a-0209f019-202402290000.log + jobid: 541 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402290000.ok + wildcards: run_id=b30a-0209f019, init_time=202402290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401040000.ok + log: output/logs/execute_inference/b30a-0209f019-202401040000.log + jobid: 373 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401040000.ok + wildcards: run_id=b30a-0209f019, init_time=202401040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404250000.ok + log: output/logs/execute_inference/b30a-0209f019-202404250000.log + jobid: 709 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202404250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407180000.ok + log: output/logs/execute_inference/b30a-0209f019-202407180000.log + jobid: 961 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407180000.ok + wildcards: run_id=b30a-0209f019, init_time=202407180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411070000.ok + log: output/logs/execute_inference/b30a-0209f019-202411070000.log + jobid: 1297 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411070000.ok + wildcards: run_id=b30a-0209f019, init_time=202411070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410100000.ok + log: output/logs/execute_inference/b30a-0209f019-202410100000.log + jobid: 1213 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202410100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409130000.ok + log: output/logs/execute_inference/b30a-0209f019-202409130000.log + jobid: 1132 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409130000.ok + wildcards: run_id=b30a-0209f019, init_time=202409130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402020000.ok + log: output/logs/execute_inference/b30a-0209f019-202402020000.log + jobid: 460 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202402020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408160000.ok + log: output/logs/execute_inference/b30a-0209f019-202408160000.log + jobid: 1048 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408160000.ok + wildcards: run_id=b30a-0209f019, init_time=202408160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403010000.ok + log: output/logs/execute_inference/b30a-0209f019-202403010000.log + jobid: 544 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403010000.ok + wildcards: run_id=b30a-0209f019, init_time=202403010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401050000.ok + log: output/logs/execute_inference/b30a-0209f019-202401050000.log + jobid: 376 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401050000.ok + wildcards: run_id=b30a-0209f019, init_time=202401050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404260000.ok + log: output/logs/execute_inference/b30a-0209f019-202404260000.log + jobid: 712 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404260000.ok + wildcards: run_id=b30a-0209f019, init_time=202404260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411080000.ok + log: output/logs/execute_inference/b30a-0209f019-202411080000.log + jobid: 1300 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202411080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407190000.ok + log: output/logs/execute_inference/b30a-0209f019-202407190000.log + jobid: 964 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202407190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_aggregation_baseline: + input: workflow/scripts/verif_aggregation.py, output/data/baselines/ICON-CH1-EPS/202401020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412250000/verif.nc + output: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + log: output/logs/verif_metrics_aggregation_baseline/ICON-CH1-EPS.log + jobid: 2 + reason: Input files updated by another job: output/data/baselines/ICON-CH1-EPS/202407200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412250000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409190000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404210000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406160000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412220000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407040000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410260000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406230000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407010000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409120000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407100000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408130000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401110000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409270000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404280000/verif.nc, output/data/baselines/ICON-CH1-EPS/202406170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410060000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401300000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407030000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401240000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202403080000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402200000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202402150000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408050000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410170000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401290000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401310000/verif.nc, output/data/baselines/ICON-CH1-EPS/202410180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202405140000/verif.nc, output/data/baselines/ICON-CH1-EPS/202411180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202408070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202409090000/verif.nc, output/data/baselines/ICON-CH1-EPS/202404180000/verif.nc, output/data/baselines/ICON-CH1-EPS/202407070000/verif.nc, output/data/baselines/ICON-CH1-EPS/202401020000/verif.nc, output/data/baselines/ICON-CH1-EPS/202412120000/verif.nc + wildcards: baseline_id=ICON-CH1-EPS + resources: mem_mb=250000, mem_mib=238419, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=1440, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410110000.ok + log: output/logs/execute_inference/b30a-0209f019-202410110000.log + jobid: 1216 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202410110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403020000.ok + log: output/logs/execute_inference/b30a-0209f019-202403020000.log + jobid: 547 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403020000.ok + wildcards: run_id=b30a-0209f019, init_time=202403020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409140000.ok + log: output/logs/execute_inference/b30a-0209f019-202409140000.log + jobid: 1135 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202409140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402030000.ok + log: output/logs/execute_inference/b30a-0209f019-202402030000.log + jobid: 463 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402030000.ok + wildcards: run_id=b30a-0209f019, init_time=202402030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408170000.ok + log: output/logs/execute_inference/b30a-0209f019-202408170000.log + jobid: 1051 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408170000.ok + wildcards: run_id=b30a-0209f019, init_time=202408170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404270000.ok + log: output/logs/execute_inference/b30a-0209f019-202404270000.log + jobid: 715 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404270000.ok + wildcards: run_id=b30a-0209f019, init_time=202404270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407200000.ok + log: output/logs/execute_inference/b30a-0209f019-202407200000.log + jobid: 967 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407200000.ok + wildcards: run_id=b30a-0209f019, init_time=202407200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401060000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401060000.ok + log: output/logs/execute_inference/b30a-0209f019-202401060000.log + jobid: 379 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401060000.ok + wildcards: run_id=b30a-0209f019, init_time=202401060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411090000.ok + log: output/logs/execute_inference/b30a-0209f019-202411090000.log + jobid: 1303 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411090000.ok + wildcards: run_id=b30a-0209f019, init_time=202411090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410120000.ok + log: output/logs/execute_inference/b30a-0209f019-202410120000.log + jobid: 1219 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410120000.ok + wildcards: run_id=b30a-0209f019, init_time=202410120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409150000.ok + log: output/logs/execute_inference/b30a-0209f019-202409150000.log + jobid: 1138 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202409150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402040000.ok + log: output/logs/execute_inference/b30a-0209f019-202402040000.log + jobid: 466 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402040000.ok + wildcards: run_id=b30a-0209f019, init_time=202402040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408180000.ok + log: output/logs/execute_inference/b30a-0209f019-202408180000.log + jobid: 1054 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408180000.ok + wildcards: run_id=b30a-0209f019, init_time=202408180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403030000.ok + log: output/logs/execute_inference/b30a-0209f019-202403030000.log + jobid: 550 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403030000.ok + wildcards: run_id=b30a-0209f019, init_time=202403030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404280000.ok + log: output/logs/execute_inference/b30a-0209f019-202404280000.log + jobid: 718 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404280000.ok + wildcards: run_id=b30a-0209f019, init_time=202404280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407210000.ok + log: output/logs/execute_inference/b30a-0209f019-202407210000.log + jobid: 970 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202407210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410130000.ok + log: output/logs/execute_inference/b30a-0209f019-202410130000.log + jobid: 1222 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410130000.ok + wildcards: run_id=b30a-0209f019, init_time=202410130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401140000.ok + log: output/logs/execute_inference/b30a-0209f019-202401140000.log + jobid: 403 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202401140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411100000.ok + log: output/logs/execute_inference/b30a-0209f019-202411100000.log + jobid: 1306 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202411100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411110000.ok + log: output/logs/execute_inference/b30a-0209f019-202411110000.log + jobid: 1309 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411110000.ok + wildcards: run_id=b30a-0209f019, init_time=202411110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401070000.ok + log: output/logs/execute_inference/b30a-0209f019-202401070000.log + jobid: 382 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401070000.ok + wildcards: run_id=b30a-0209f019, init_time=202401070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409160000.ok + log: output/logs/execute_inference/b30a-0209f019-202409160000.log + jobid: 1141 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409160000.ok + wildcards: run_id=b30a-0209f019, init_time=202409160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402050000.ok + log: output/logs/execute_inference/b30a-0209f019-202402050000.log + jobid: 469 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202402050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408190000.ok + log: output/logs/execute_inference/b30a-0209f019-202408190000.log + jobid: 1057 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408190000.ok + wildcards: run_id=b30a-0209f019, init_time=202408190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403040000.ok + log: output/logs/execute_inference/b30a-0209f019-202403040000.log + jobid: 553 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403040000.ok + wildcards: run_id=b30a-0209f019, init_time=202403040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404290000.ok + log: output/logs/execute_inference/b30a-0209f019-202404290000.log + jobid: 721 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404290000.ok + wildcards: run_id=b30a-0209f019, init_time=202404290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407220000.ok + log: output/logs/execute_inference/b30a-0209f019-202407220000.log + jobid: 973 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407220000.ok + wildcards: run_id=b30a-0209f019, init_time=202407220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410140000.ok + log: output/logs/execute_inference/b30a-0209f019-202410140000.log + jobid: 1225 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202410140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401130000.ok + log: output/logs/execute_inference/b30a-0209f019-202401130000.log + jobid: 400 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401130000.ok + wildcards: run_id=b30a-0209f019, init_time=202401130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403050000.ok + log: output/logs/execute_inference/b30a-0209f019-202403050000.log + jobid: 556 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402060000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402060000.ok + log: output/logs/execute_inference/b30a-0209f019-202402060000.log + jobid: 472 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402060000.ok + wildcards: run_id=b30a-0209f019, init_time=202402060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409170000.ok + log: output/logs/execute_inference/b30a-0209f019-202409170000.log + jobid: 1144 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409170000.ok + wildcards: run_id=b30a-0209f019, init_time=202409170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408200000.ok + log: output/logs/execute_inference/b30a-0209f019-202408200000.log + jobid: 1060 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202408200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404300000.ok + log: output/logs/execute_inference/b30a-0209f019-202404300000.log + jobid: 724 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404300000.ok + wildcards: run_id=b30a-0209f019, init_time=202404300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407230000.ok + log: output/logs/execute_inference/b30a-0209f019-202407230000.log + jobid: 976 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202407230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410150000.ok + log: output/logs/execute_inference/b30a-0209f019-202410150000.log + jobid: 1228 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410150000.ok + wildcards: run_id=b30a-0209f019, init_time=202410150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411120000.ok + log: output/logs/execute_inference/b30a-0209f019-202411120000.log + jobid: 1312 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202411120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401120000.ok + log: output/logs/execute_inference/b30a-0209f019-202401120000.log + jobid: 397 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202401120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401080000.ok + log: output/logs/execute_inference/b30a-0209f019-202401080000.log + jobid: 385 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202401080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408210000.ok + log: output/logs/execute_inference/b30a-0209f019-202408210000.log + jobid: 1063 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408210000.ok + wildcards: run_id=b30a-0209f019, init_time=202408210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402070000.ok + log: output/logs/execute_inference/b30a-0209f019-202402070000.log + jobid: 475 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402070000.ok + wildcards: run_id=b30a-0209f019, init_time=202402070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409180000.ok + log: output/logs/execute_inference/b30a-0209f019-202409180000.log + jobid: 1147 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409180000.ok + wildcards: run_id=b30a-0209f019, init_time=202409180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403060000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403060000.ok + log: output/logs/execute_inference/b30a-0209f019-202403060000.log + jobid: 559 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403060000.ok + wildcards: run_id=b30a-0209f019, init_time=202403060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407240000.ok + log: output/logs/execute_inference/b30a-0209f019-202407240000.log + jobid: 979 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202407240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405010000.ok + log: output/logs/execute_inference/b30a-0209f019-202405010000.log + jobid: 727 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202405010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410160000.ok + log: output/logs/execute_inference/b30a-0209f019-202410160000.log + jobid: 1231 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202410160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411130000.ok + log: output/logs/execute_inference/b30a-0209f019-202411130000.log + jobid: 1315 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411130000.ok + wildcards: run_id=b30a-0209f019, init_time=202411130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401090000.ok + log: output/logs/execute_inference/b30a-0209f019-202401090000.log + jobid: 388 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401090000.ok + wildcards: run_id=b30a-0209f019, init_time=202401090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402080000.ok + log: output/logs/execute_inference/b30a-0209f019-202402080000.log + jobid: 478 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402080000.ok + wildcards: run_id=b30a-0209f019, init_time=202402080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409190000.ok + log: output/logs/execute_inference/b30a-0209f019-202409190000.log + jobid: 1150 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409190000.ok + wildcards: run_id=b30a-0209f019, init_time=202409190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408220000.ok + log: output/logs/execute_inference/b30a-0209f019-202408220000.log + jobid: 1066 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408220000.ok + wildcards: run_id=b30a-0209f019, init_time=202408220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403070000.ok + log: output/logs/execute_inference/b30a-0209f019-202403070000.log + jobid: 562 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405020000.ok + log: output/logs/execute_inference/b30a-0209f019-202405020000.log + jobid: 730 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202405020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407250000.ok + log: output/logs/execute_inference/b30a-0209f019-202407250000.log + jobid: 982 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407250000.ok + wildcards: run_id=b30a-0209f019, init_time=202407250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410170000.ok + log: output/logs/execute_inference/b30a-0209f019-202410170000.log + jobid: 1234 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410170000.ok + wildcards: run_id=b30a-0209f019, init_time=202410170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411140000.ok + log: output/logs/execute_inference/b30a-0209f019-202411140000.log + jobid: 1318 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411140000.ok + wildcards: run_id=b30a-0209f019, init_time=202411140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401100000.ok + log: output/logs/execute_inference/b30a-0209f019-202401100000.log + jobid: 391 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401100000.ok + wildcards: run_id=b30a-0209f019, init_time=202401100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411150000.ok + log: output/logs/execute_inference/b30a-0209f019-202411150000.log + jobid: 1321 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411150000.ok + wildcards: run_id=b30a-0209f019, init_time=202411150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407260000.ok + log: output/logs/execute_inference/b30a-0209f019-202407260000.log + jobid: 985 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202407260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410180000.ok + log: output/logs/execute_inference/b30a-0209f019-202410180000.log + jobid: 1237 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410180000.ok + wildcards: run_id=b30a-0209f019, init_time=202410180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403080000.ok + log: output/logs/execute_inference/b30a-0209f019-202403080000.log + jobid: 565 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408230000.ok + log: output/logs/execute_inference/b30a-0209f019-202408230000.log + jobid: 1069 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202408230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402090000.ok + log: output/logs/execute_inference/b30a-0209f019-202402090000.log + jobid: 481 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402090000.ok + wildcards: run_id=b30a-0209f019, init_time=202402090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409200000.ok + log: output/logs/execute_inference/b30a-0209f019-202409200000.log + jobid: 1153 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202409200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405030000.ok + log: output/logs/execute_inference/b30a-0209f019-202405030000.log + jobid: 733 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405030000.ok + wildcards: run_id=b30a-0209f019, init_time=202405030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401110000.ok + log: output/logs/execute_inference/b30a-0209f019-202401110000.log + jobid: 394 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401110000.ok + wildcards: run_id=b30a-0209f019, init_time=202401110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403090000.ok + log: output/logs/execute_inference/b30a-0209f019-202403090000.log + jobid: 568 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403090000.ok + wildcards: run_id=b30a-0209f019, init_time=202403090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408240000.ok + log: output/logs/execute_inference/b30a-0209f019-202408240000.log + jobid: 1072 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408240000.ok + wildcards: run_id=b30a-0209f019, init_time=202408240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402100000.ok + log: output/logs/execute_inference/b30a-0209f019-202402100000.log + jobid: 484 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402100000.ok + wildcards: run_id=b30a-0209f019, init_time=202402100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409210000.ok + log: output/logs/execute_inference/b30a-0209f019-202409210000.log + jobid: 1156 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409210000.ok + wildcards: run_id=b30a-0209f019, init_time=202409210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405040000.ok + log: output/logs/execute_inference/b30a-0209f019-202405040000.log + jobid: 736 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405040000.ok + wildcards: run_id=b30a-0209f019, init_time=202405040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407270000.ok + log: output/logs/execute_inference/b30a-0209f019-202407270000.log + jobid: 988 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202407270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202407270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411160000.ok + log: output/logs/execute_inference/b30a-0209f019-202411160000.log + jobid: 1324 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411160000.ok + wildcards: run_id=b30a-0209f019, init_time=202411160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410190000.ok + log: output/logs/execute_inference/b30a-0209f019-202410190000.log + jobid: 1240 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410190000.ok + wildcards: run_id=b30a-0209f019, init_time=202410190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403100000.ok + log: output/logs/execute_inference/b30a-0209f019-202403100000.log + jobid: 571 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408250000.ok + log: output/logs/execute_inference/b30a-0209f019-202408250000.log + jobid: 1075 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202408250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402110000.ok + log: output/logs/execute_inference/b30a-0209f019-202402110000.log + jobid: 487 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202402110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202402110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409220000.ok + log: output/logs/execute_inference/b30a-0209f019-202409220000.log + jobid: 1159 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202409220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407280000.ok + log: output/logs/execute_inference/b30a-0209f019-202407280000.log + jobid: 991 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407280000.ok + wildcards: run_id=b30a-0209f019, init_time=202407280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405050000.ok + log: output/logs/execute_inference/b30a-0209f019-202405050000.log + jobid: 739 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405050000.ok + wildcards: run_id=b30a-0209f019, init_time=202405050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411170000.ok + log: output/logs/execute_inference/b30a-0209f019-202411170000.log + jobid: 1327 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202411170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410200000.ok + log: output/logs/execute_inference/b30a-0209f019-202410200000.log + jobid: 1243 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410200000.ok + wildcards: run_id=b30a-0209f019, init_time=202410200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410210000.ok + log: output/logs/execute_inference/b30a-0209f019-202410210000.log + jobid: 1246 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202410210000.ok + wildcards: run_id=b30a-0209f019, init_time=202410210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403110000.ok + log: output/logs/execute_inference/b30a-0209f019-202403110000.log + jobid: 574 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403110000.ok + wildcards: run_id=b30a-0209f019, init_time=202403110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408260000.ok + log: output/logs/execute_inference/b30a-0209f019-202408260000.log + jobid: 1078 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408260000.ok + wildcards: run_id=b30a-0209f019, init_time=202408260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402120000.ok + log: output/logs/execute_inference/b30a-0209f019-202402120000.log + jobid: 490 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402120000.ok + wildcards: run_id=b30a-0209f019, init_time=202402120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409230000.ok + log: output/logs/execute_inference/b30a-0209f019-202409230000.log + jobid: 1162 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202409230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405060000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405060000.ok + log: output/logs/execute_inference/b30a-0209f019-202405060000.log + jobid: 742 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405060000.ok + wildcards: run_id=b30a-0209f019, init_time=202405060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407290000.ok + log: output/logs/execute_inference/b30a-0209f019-202407290000.log + jobid: 994 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407290000.ok + wildcards: run_id=b30a-0209f019, init_time=202407290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401150000.ok + log: output/logs/execute_inference/b30a-0209f019-202401150000.log + jobid: 406 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202401150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202401150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411180000.ok + log: output/logs/execute_inference/b30a-0209f019-202411180000.log + jobid: 1330 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202411180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410220000.ok + log: output/logs/execute_inference/b30a-0209f019-202410220000.log + jobid: 1249 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202410220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411190000.ok + log: output/logs/execute_inference/b30a-0209f019-202411190000.log + jobid: 1333 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202411190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409240000.ok + log: output/logs/execute_inference/b30a-0209f019-202409240000.log + jobid: 1165 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202409240000.ok + wildcards: run_id=b30a-0209f019, init_time=202409240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408270000.ok + log: output/logs/execute_inference/b30a-0209f019-202408270000.log + jobid: 1081 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408270000.ok + wildcards: run_id=b30a-0209f019, init_time=202408270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402130000.ok + log: output/logs/execute_inference/b30a-0209f019-202402130000.log + jobid: 493 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402130000.ok + wildcards: run_id=b30a-0209f019, init_time=202402130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403120000.ok + log: output/logs/execute_inference/b30a-0209f019-202403120000.log + jobid: 577 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405070000.ok + log: output/logs/execute_inference/b30a-0209f019-202405070000.log + jobid: 745 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202405070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407300000.ok + log: output/logs/execute_inference/b30a-0209f019-202407300000.log + jobid: 997 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407300000.ok + wildcards: run_id=b30a-0209f019, init_time=202407300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401160000.ok + log: output/logs/execute_inference/b30a-0209f019-202401160000.log + jobid: 409 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401160000.ok + wildcards: run_id=b30a-0209f019, init_time=202401160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410230000.ok + log: output/logs/execute_inference/b30a-0209f019-202410230000.log + jobid: 1252 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202410230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411200000.ok + log: output/logs/execute_inference/b30a-0209f019-202411200000.log + jobid: 1336 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411200000.ok + wildcards: run_id=b30a-0209f019, init_time=202411200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403130000.ok + log: output/logs/execute_inference/b30a-0209f019-202403130000.log + jobid: 580 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403130000.ok + wildcards: run_id=b30a-0209f019, init_time=202403130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405080000.ok + log: output/logs/execute_inference/b30a-0209f019-202405080000.log + jobid: 748 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405080000.ok + wildcards: run_id=b30a-0209f019, init_time=202405080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402140000.ok + log: output/logs/execute_inference/b30a-0209f019-202402140000.log + jobid: 496 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402140000.ok + wildcards: run_id=b30a-0209f019, init_time=202402140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401170000.ok + log: output/logs/execute_inference/b30a-0209f019-202401170000.log + jobid: 412 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401170000.ok + wildcards: run_id=b30a-0209f019, init_time=202401170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407310000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407310000.ok + log: output/logs/execute_inference/b30a-0209f019-202407310000.log + jobid: 1000 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407310000.ok + wildcards: run_id=b30a-0209f019, init_time=202407310000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408280000.ok + log: output/logs/execute_inference/b30a-0209f019-202408280000.log + jobid: 1084 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202408280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409250000.ok + log: output/logs/execute_inference/b30a-0209f019-202409250000.log + jobid: 1168 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202409250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202410240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202410240000.ok + log: output/logs/execute_inference/b30a-0209f019-202410240000.log + jobid: 1255 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202410240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202410240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202409260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202409260000.ok + log: output/logs/execute_inference/b30a-0209f019-202409260000.log + jobid: 1171 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202409260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202409260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405090000.ok + log: output/logs/execute_inference/b30a-0209f019-202405090000.log + jobid: 751 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202405090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411210000.ok + log: output/logs/execute_inference/b30a-0209f019-202411210000.log + jobid: 1339 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411210000.ok + wildcards: run_id=b30a-0209f019, init_time=202411210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403140000.ok + log: output/logs/execute_inference/b30a-0209f019-202403140000.log + jobid: 583 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401180000.ok + log: output/logs/execute_inference/b30a-0209f019-202401180000.log + jobid: 415 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202401180000.ok + wildcards: run_id=b30a-0209f019, init_time=202401180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202402150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202402150000.ok + log: output/logs/execute_inference/b30a-0209f019-202402150000.log + jobid: 499 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202402150000.ok + wildcards: run_id=b30a-0209f019, init_time=202402150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408010000.ok + log: output/logs/execute_inference/b30a-0209f019-202408010000.log + jobid: 1003 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202408010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202408010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202408290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202408290000.ok + log: output/logs/execute_inference/b30a-0209f019-202408290000.log + jobid: 1087 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202408290000.ok + wildcards: run_id=b30a-0209f019, init_time=202408290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403150000.ok + log: output/logs/execute_inference/b30a-0209f019-202403150000.log + jobid: 586 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403150000.ok + wildcards: run_id=b30a-0209f019, init_time=202403150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405100000.ok + log: output/logs/execute_inference/b30a-0209f019-202405100000.log + jobid: 754 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405100000.ok + wildcards: run_id=b30a-0209f019, init_time=202405100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412200000.ok + log: output/logs/execute_inference/b30a-0209f019-202412200000.log + jobid: 1426 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412200000.ok + wildcards: run_id=b30a-0209f019, init_time=202412200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411220000.ok + log: output/logs/execute_inference/b30a-0209f019-202411220000.log + jobid: 1342 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411220000.ok + wildcards: run_id=b30a-0209f019, init_time=202411220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406070000.ok + log: output/logs/execute_inference/b30a-0209f019-202406070000.log + jobid: 838 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403160000.ok + log: output/logs/execute_inference/b30a-0209f019-202403160000.log + jobid: 589 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403160000.ok + wildcards: run_id=b30a-0209f019, init_time=202403160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411230000.ok + log: output/logs/execute_inference/b30a-0209f019-202411230000.log + jobid: 1345 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411230000.ok + wildcards: run_id=b30a-0209f019, init_time=202411230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406080000.ok + log: output/logs/execute_inference/b30a-0209f019-202406080000.log + jobid: 841 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406080000.ok + wildcards: run_id=b30a-0209f019, init_time=202406080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405110000.ok + log: output/logs/execute_inference/b30a-0209f019-202405110000.log + jobid: 757 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202405110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412210000.ok + log: output/logs/execute_inference/b30a-0209f019-202412210000.log + jobid: 1429 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412210000.ok + wildcards: run_id=b30a-0209f019, init_time=202412210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412220000.ok + log: output/logs/execute_inference/b30a-0209f019-202412220000.log + jobid: 1432 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412220000.ok + wildcards: run_id=b30a-0209f019, init_time=202412220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403170000.ok + log: output/logs/execute_inference/b30a-0209f019-202403170000.log + jobid: 592 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411240000.ok + log: output/logs/execute_inference/b30a-0209f019-202411240000.log + jobid: 1348 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411240000.ok + wildcards: run_id=b30a-0209f019, init_time=202411240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406090000.ok + log: output/logs/execute_inference/b30a-0209f019-202406090000.log + jobid: 844 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405120000.ok + log: output/logs/execute_inference/b30a-0209f019-202405120000.log + jobid: 760 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405120000.ok + wildcards: run_id=b30a-0209f019, init_time=202405120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403180000.ok + log: output/logs/execute_inference/b30a-0209f019-202403180000.log + jobid: 595 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403180000.ok + wildcards: run_id=b30a-0209f019, init_time=202403180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405130000.ok + log: output/logs/execute_inference/b30a-0209f019-202405130000.log + jobid: 763 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405130000.ok + wildcards: run_id=b30a-0209f019, init_time=202405130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411250000.ok + log: output/logs/execute_inference/b30a-0209f019-202411250000.log + jobid: 1351 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202411250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412230000.ok + log: output/logs/execute_inference/b30a-0209f019-202412230000.log + jobid: 1435 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412230000.ok + wildcards: run_id=b30a-0209f019, init_time=202412230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406100000.ok + log: output/logs/execute_inference/b30a-0209f019-202406100000.log + jobid: 847 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406100000.ok + wildcards: run_id=b30a-0209f019, init_time=202406100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411260000.ok + log: output/logs/execute_inference/b30a-0209f019-202411260000.log + jobid: 1354 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411260000.ok + wildcards: run_id=b30a-0209f019, init_time=202411260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403190000.ok + log: output/logs/execute_inference/b30a-0209f019-202403190000.log + jobid: 598 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406110000.ok + log: output/logs/execute_inference/b30a-0209f019-202406110000.log + jobid: 850 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406110000.ok + wildcards: run_id=b30a-0209f019, init_time=202406110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412240000.ok + log: output/logs/execute_inference/b30a-0209f019-202412240000.log + jobid: 1438 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202412240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405140000.ok + log: output/logs/execute_inference/b30a-0209f019-202405140000.log + jobid: 766 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405140000.ok + wildcards: run_id=b30a-0209f019, init_time=202405140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405150000.ok + log: output/logs/execute_inference/b30a-0209f019-202405150000.log + jobid: 769 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405150000.ok + wildcards: run_id=b30a-0209f019, init_time=202405150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412250000.ok + log: output/logs/execute_inference/b30a-0209f019-202412250000.log + jobid: 1441 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202412250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403200000.ok + log: output/logs/execute_inference/b30a-0209f019-202403200000.log + jobid: 601 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411270000.ok + log: output/logs/execute_inference/b30a-0209f019-202411270000.log + jobid: 1357 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202411270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406120000.ok + log: output/logs/execute_inference/b30a-0209f019-202406120000.log + jobid: 853 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403210000.ok + log: output/logs/execute_inference/b30a-0209f019-202403210000.log + jobid: 604 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405160000.ok + log: output/logs/execute_inference/b30a-0209f019-202405160000.log + jobid: 772 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202405160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406130000.ok + log: output/logs/execute_inference/b30a-0209f019-202406130000.log + jobid: 856 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411280000.ok + log: output/logs/execute_inference/b30a-0209f019-202411280000.log + jobid: 1360 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202411280000.ok + wildcards: run_id=b30a-0209f019, init_time=202411280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403220000.ok + log: output/logs/execute_inference/b30a-0209f019-202403220000.log + jobid: 607 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411290000.ok + log: output/logs/execute_inference/b30a-0209f019-202411290000.log + jobid: 1363 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202411290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406140000.ok + log: output/logs/execute_inference/b30a-0209f019-202406140000.log + jobid: 859 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406140000.ok + wildcards: run_id=b30a-0209f019, init_time=202406140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405170000.ok + log: output/logs/execute_inference/b30a-0209f019-202405170000.log + jobid: 775 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202405170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406150000.ok + log: output/logs/execute_inference/b30a-0209f019-202406150000.log + jobid: 862 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406150000.ok + wildcards: run_id=b30a-0209f019, init_time=202406150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405180000.ok + log: output/logs/execute_inference/b30a-0209f019-202405180000.log + jobid: 778 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405180000.ok + wildcards: run_id=b30a-0209f019, init_time=202405180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403230000.ok + log: output/logs/execute_inference/b30a-0209f019-202403230000.log + jobid: 610 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403230000.ok + wildcards: run_id=b30a-0209f019, init_time=202403230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202411300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202411300000.ok + log: output/logs/execute_inference/b30a-0209f019-202411300000.log + jobid: 1366 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202411300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202411300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406160000.ok + log: output/logs/execute_inference/b30a-0209f019-202406160000.log + jobid: 865 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406160000.ok + wildcards: run_id=b30a-0209f019, init_time=202406160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405190000.ok + log: output/logs/execute_inference/b30a-0209f019-202405190000.log + jobid: 781 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405190000.ok + wildcards: run_id=b30a-0209f019, init_time=202405190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403240000.ok + log: output/logs/execute_inference/b30a-0209f019-202403240000.log + jobid: 613 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412010000.ok + log: output/logs/execute_inference/b30a-0209f019-202412010000.log + jobid: 1369 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412010000.ok + wildcards: run_id=b30a-0209f019, init_time=202412010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412020000.ok + log: output/logs/execute_inference/b30a-0209f019-202412020000.log + jobid: 1372 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412020000.ok + wildcards: run_id=b30a-0209f019, init_time=202412020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405200000.ok + log: output/logs/execute_inference/b30a-0209f019-202405200000.log + jobid: 784 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405200000.ok + wildcards: run_id=b30a-0209f019, init_time=202405200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406170000.ok + log: output/logs/execute_inference/b30a-0209f019-202406170000.log + jobid: 868 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403250000.ok + log: output/logs/execute_inference/b30a-0209f019-202403250000.log + jobid: 616 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405210000.ok + log: output/logs/execute_inference/b30a-0209f019-202405210000.log + jobid: 787 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405210000.ok + wildcards: run_id=b30a-0209f019, init_time=202405210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406180000.ok + log: output/logs/execute_inference/b30a-0209f019-202406180000.log + jobid: 871 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406180000.ok + wildcards: run_id=b30a-0209f019, init_time=202406180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412030000.ok + log: output/logs/execute_inference/b30a-0209f019-202412030000.log + jobid: 1375 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202412030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403260000.ok + log: output/logs/execute_inference/b30a-0209f019-202403260000.log + jobid: 619 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403260000.ok + wildcards: run_id=b30a-0209f019, init_time=202403260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403270000.ok + log: output/logs/execute_inference/b30a-0209f019-202403270000.log + jobid: 622 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403270000.ok + wildcards: run_id=b30a-0209f019, init_time=202403270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406190000.ok + log: output/logs/execute_inference/b30a-0209f019-202406190000.log + jobid: 874 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405220000.ok + log: output/logs/execute_inference/b30a-0209f019-202405220000.log + jobid: 790 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405220000.ok + wildcards: run_id=b30a-0209f019, init_time=202405220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412040000.ok + log: output/logs/execute_inference/b30a-0209f019-202412040000.log + jobid: 1378 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202412040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412050000.ok + log: output/logs/execute_inference/b30a-0209f019-202412050000.log + jobid: 1381 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412050000.ok + wildcards: run_id=b30a-0209f019, init_time=202412050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406200000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406200000.ok + log: output/logs/execute_inference/b30a-0209f019-202406200000.log + jobid: 877 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406200000.ok + wildcards: run_id=b30a-0209f019, init_time=202406200000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403280000.ok + log: output/logs/execute_inference/b30a-0209f019-202403280000.log + jobid: 625 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405230000.ok + log: output/logs/execute_inference/b30a-0209f019-202405230000.log + jobid: 793 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405230000.ok + wildcards: run_id=b30a-0209f019, init_time=202405230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403290000.ok + log: output/logs/execute_inference/b30a-0209f019-202403290000.log + jobid: 628 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405240000.ok + log: output/logs/execute_inference/b30a-0209f019-202405240000.log + jobid: 796 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405240000.ok + wildcards: run_id=b30a-0209f019, init_time=202405240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412060000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412060000.ok + log: output/logs/execute_inference/b30a-0209f019-202412060000.log + jobid: 1384 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412060000.ok + wildcards: run_id=b30a-0209f019, init_time=202412060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406210000.ok + log: output/logs/execute_inference/b30a-0209f019-202406210000.log + jobid: 880 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406210000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406210000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403300000.ok + log: output/logs/execute_inference/b30a-0209f019-202403300000.log + jobid: 631 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202403300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202403300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405250000.ok + log: output/logs/execute_inference/b30a-0209f019-202405250000.log + jobid: 799 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405250000.ok + wildcards: run_id=b30a-0209f019, init_time=202405250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412070000.ok + log: output/logs/execute_inference/b30a-0209f019-202412070000.log + jobid: 1387 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412070000.ok + wildcards: run_id=b30a-0209f019, init_time=202412070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406220000.ok + log: output/logs/execute_inference/b30a-0209f019-202406220000.log + jobid: 883 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406220000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406220000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202403310000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202403310000.ok + log: output/logs/execute_inference/b30a-0209f019-202403310000.log + jobid: 634 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202403310000.ok + wildcards: run_id=b30a-0209f019, init_time=202403310000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405260000.ok + log: output/logs/execute_inference/b30a-0209f019-202405260000.log + jobid: 802 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405260000.ok + wildcards: run_id=b30a-0209f019, init_time=202405260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412080000.ok + log: output/logs/execute_inference/b30a-0209f019-202412080000.log + jobid: 1390 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412080000.ok + wildcards: run_id=b30a-0209f019, init_time=202412080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406230000.ok + log: output/logs/execute_inference/b30a-0209f019-202406230000.log + jobid: 886 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406230000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406230000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404010000.ok + log: output/logs/execute_inference/b30a-0209f019-202404010000.log + jobid: 637 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202404010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405270000.ok + log: output/logs/execute_inference/b30a-0209f019-202405270000.log + jobid: 805 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202405270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412090000.ok + log: output/logs/execute_inference/b30a-0209f019-202412090000.log + jobid: 1393 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412090000.ok + wildcards: run_id=b30a-0209f019, init_time=202412090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406240000.ok + log: output/logs/execute_inference/b30a-0209f019-202406240000.log + jobid: 889 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406240000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406240000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404020000.ok + log: output/logs/execute_inference/b30a-0209f019-202404020000.log + jobid: 640 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404020000.ok + wildcards: run_id=b30a-0209f019, init_time=202404020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405280000.ok + log: output/logs/execute_inference/b30a-0209f019-202405280000.log + jobid: 808 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202405280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412100000.ok + log: output/logs/execute_inference/b30a-0209f019-202412100000.log + jobid: 1396 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202412100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406250000.ok + log: output/logs/execute_inference/b30a-0209f019-202406250000.log + jobid: 892 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406250000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406250000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404030000.ok + log: output/logs/execute_inference/b30a-0209f019-202404030000.log + jobid: 643 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404030000.ok + wildcards: run_id=b30a-0209f019, init_time=202404030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405290000.ok + log: output/logs/execute_inference/b30a-0209f019-202405290000.log + jobid: 811 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405290000.ok + wildcards: run_id=b30a-0209f019, init_time=202405290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412110000.ok + log: output/logs/execute_inference/b30a-0209f019-202412110000.log + jobid: 1399 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202412110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406260000.ok + log: output/logs/execute_inference/b30a-0209f019-202406260000.log + jobid: 895 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406260000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406260000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404040000.ok + log: output/logs/execute_inference/b30a-0209f019-202404040000.log + jobid: 646 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202404040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405300000.ok + log: output/logs/execute_inference/b30a-0209f019-202405300000.log + jobid: 814 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202405300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202405300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412120000.ok + log: output/logs/execute_inference/b30a-0209f019-202412120000.log + jobid: 1402 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412120000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202412120000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406270000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406270000.ok + log: output/logs/execute_inference/b30a-0209f019-202406270000.log + jobid: 898 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406270000.ok + wildcards: run_id=b30a-0209f019, init_time=202406270000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404050000.ok + log: output/logs/execute_inference/b30a-0209f019-202404050000.log + jobid: 649 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404050000.ok + wildcards: run_id=b30a-0209f019, init_time=202404050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202405310000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202405310000.ok + log: output/logs/execute_inference/b30a-0209f019-202405310000.log + jobid: 817 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202405310000.ok + wildcards: run_id=b30a-0209f019, init_time=202405310000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412130000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412130000.ok + log: output/logs/execute_inference/b30a-0209f019-202412130000.log + jobid: 1405 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412130000.ok + wildcards: run_id=b30a-0209f019, init_time=202412130000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406280000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406280000.ok + log: output/logs/execute_inference/b30a-0209f019-202406280000.log + jobid: 901 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406280000.ok + wildcards: run_id=b30a-0209f019, init_time=202406280000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404060000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404060000.ok + log: output/logs/execute_inference/b30a-0209f019-202404060000.log + jobid: 652 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404060000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202404060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406010000.ok + log: output/logs/execute_inference/b30a-0209f019-202406010000.log + jobid: 820 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406010000.ok + wildcards: run_id=b30a-0209f019, init_time=202406010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412140000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412140000.ok + log: output/logs/execute_inference/b30a-0209f019-202412140000.log + jobid: 1408 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412140000.ok + wildcards: run_id=b30a-0209f019, init_time=202412140000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406290000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406290000.ok + log: output/logs/execute_inference/b30a-0209f019-202406290000.log + jobid: 904 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406290000.ok + wildcards: run_id=b30a-0209f019, init_time=202406290000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404070000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404070000.ok + log: output/logs/execute_inference/b30a-0209f019-202404070000.log + jobid: 655 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404070000.ok + wildcards: run_id=b30a-0209f019, init_time=202404070000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406020000.ok + log: output/logs/execute_inference/b30a-0209f019-202406020000.log + jobid: 823 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412150000.ok + log: output/logs/execute_inference/b30a-0209f019-202412150000.log + jobid: 1411 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412150000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202412150000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406300000.ok + log: output/logs/execute_inference/b30a-0209f019-202406300000.log + jobid: 907 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406300000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406300000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404080000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404080000.ok + log: output/logs/execute_inference/b30a-0209f019-202404080000.log + jobid: 658 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404080000.ok + wildcards: run_id=b30a-0209f019, init_time=202404080000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406030000.ok + log: output/logs/execute_inference/b30a-0209f019-202406030000.log + jobid: 826 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412160000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412160000.ok + log: output/logs/execute_inference/b30a-0209f019-202412160000.log + jobid: 1414 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412160000.ok + wildcards: run_id=b30a-0209f019, init_time=202412160000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407010000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407010000.ok + log: output/logs/execute_inference/b30a-0209f019-202407010000.log + jobid: 910 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407010000.ok + wildcards: run_id=b30a-0209f019, init_time=202407010000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404090000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404090000.ok + log: output/logs/execute_inference/b30a-0209f019-202404090000.log + jobid: 661 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404090000.ok + wildcards: run_id=b30a-0209f019, init_time=202404090000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406040000.ok + log: output/logs/execute_inference/b30a-0209f019-202406040000.log + jobid: 829 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412170000.ok + log: output/logs/execute_inference/b30a-0209f019-202412170000.log + jobid: 1417 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202412170000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202412170000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407020000.ok + log: output/logs/execute_inference/b30a-0209f019-202407020000.log + jobid: 913 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407020000.ok + wildcards: run_id=b30a-0209f019, init_time=202407020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406050000.ok + log: output/logs/execute_inference/b30a-0209f019-202406050000.log + jobid: 832 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202406050000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202406050000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404100000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404100000.ok + log: output/logs/execute_inference/b30a-0209f019-202404100000.log + jobid: 664 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202404100000.ok + wildcards: run_id=b30a-0209f019, init_time=202404100000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407030000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407030000.ok + log: output/logs/execute_inference/b30a-0209f019-202407030000.log + jobid: 916 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407030000.ok + wildcards: run_id=b30a-0209f019, init_time=202407030000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412180000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412180000.ok + log: output/logs/execute_inference/b30a-0209f019-202412180000.log + jobid: 1420 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412180000.ok + wildcards: run_id=b30a-0209f019, init_time=202412180000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202401020000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202401020000.ok + log: output/logs/execute_inference/b30a-0209f019-202401020000.log + jobid: 364 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202401020000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202412190000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202412190000.ok + log: output/logs/execute_inference/b30a-0209f019-202412190000.log + jobid: 1423 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202412190000.ok + wildcards: run_id=b30a-0209f019, init_time=202412190000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202406060000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202406060000.ok + log: output/logs/execute_inference/b30a-0209f019-202406060000.log + jobid: 835 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202406060000.ok + wildcards: run_id=b30a-0209f019, init_time=202406060000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202404110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202404110000.ok + log: output/logs/execute_inference/b30a-0209f019-202404110000.log + jobid: 667 + reason: Input files updated by another job: output/logs/prepare_inference_forecaster/b30a-0209f019-202404110000.ok, output/data/runs/b30a-0209f019/venv.squashfs + wildcards: run_id=b30a-0209f019, init_time=202404110000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +[Thu Feb 26 12:18:24 2026] +rule execute_inference: + input: output/logs/prepare_inference_forecaster/b30a-0209f019-202407040000.ok, output/data/runs/b30a-0209f019/venv.squashfs + output: output/logs/execute_inference/b30a-0209f019-202407040000.ok + log: output/logs/execute_inference/b30a-0209f019-202407040000.log + jobid: 919 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/venv.squashfs, output/logs/prepare_inference_forecaster/b30a-0209f019-202407040000.ok + wildcards: run_id=b30a-0209f019, init_time=202407040000 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=short-shared, cpus_per_task=24, mem_mb_per_cpu=8000, runtime=40, gres=gpu:1, ntasks=1, gpus=1 +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401020000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401030000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401040000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401050000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401060000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401070000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401080000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401090000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401100000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401110000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401120000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401130000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401140000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401150000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401160000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401170000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401180000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401190000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401200000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401210000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401220000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401230000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401240000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401250000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401260000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401270000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401280000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401290000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401300000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202401310000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402010000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402020000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402030000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402040000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402050000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402060000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402070000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402080000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402090000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402100000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402110000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402120000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402130000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402140000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402150000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402160000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402170000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402180000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402190000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402200000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402210000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402220000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402230000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402240000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402250000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402260000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402270000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402280000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202402290000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403010000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403020000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403030000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403040000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403050000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403060000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403070000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403080000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403090000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403100000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403110000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403120000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403130000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403140000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403150000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403160000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403170000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403180000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403190000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403200000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403210000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403220000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403230000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403240000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403250000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403260000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403270000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403280000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403290000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403300000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202403310000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404010000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404020000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404030000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404040000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404050000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404060000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404070000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404080000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404090000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404100000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404110000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404120000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404130000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404140000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404150000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404160000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404170000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404180000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404190000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404200000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404210000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404220000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404230000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404240000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404250000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404260000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404270000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404280000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404290000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202404300000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405010000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405020000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405030000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405040000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405050000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405060000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405070000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405080000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405090000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405100000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405110000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405120000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405130000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405140000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405150000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405160000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405170000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405180000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405190000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405200000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405210000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405220000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405230000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405240000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405250000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405260000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405270000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405280000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405290000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405300000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202405310000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406010000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406020000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406030000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406040000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406050000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406060000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406070000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406080000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406090000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406100000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406110000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406120000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406130000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406140000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406150000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406160000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406170000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406180000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406190000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406200000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406210000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406220000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406230000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406240000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406250000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406260000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406270000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406280000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406290000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202406300000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407010000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407020000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407030000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407040000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407050000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407060000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407070000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407080000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407090000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407100000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407110000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407120000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407130000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407140000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407150000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407160000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407170000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407180000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407190000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407200000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407210000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407220000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407230000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407240000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407250000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407260000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407270000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407280000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407290000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407300000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202407310000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408010000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408020000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408030000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408040000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408050000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408060000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408070000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408080000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408090000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408100000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408110000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408120000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408130000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408140000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408150000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408160000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408170000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408180000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408190000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408200000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408210000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408220000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408230000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408240000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408250000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408260000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408270000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408280000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408290000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408300000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202408310000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409010000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409020000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409030000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409040000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409050000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409060000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409070000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409080000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409090000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409100000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409110000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409120000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409130000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409140000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409150000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409160000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409170000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409180000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409190000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409200000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409210000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409220000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409230000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409240000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409250000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409260000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409270000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409280000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409290000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202409300000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410010000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410020000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410030000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410040000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410050000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410060000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410070000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410080000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410090000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410100000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410110000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410120000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410130000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410140000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410150000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410160000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410170000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410180000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410190000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410200000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410210000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410220000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410230000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410240000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410250000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410260000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410270000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410280000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410290000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410300000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202410310000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411010000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411020000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411030000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411040000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411050000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411060000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411070000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411080000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411090000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411100000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411110000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411120000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411130000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411140000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411150000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411160000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411170000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411180000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411190000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411200000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411210000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411220000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411230000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411240000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411250000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411260000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411270000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411280000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411290000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202411300000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412010000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412020000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412030000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412040000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412050000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412060000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412070000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412080000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412090000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412100000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412110000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412120000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412130000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412140000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412150000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412160000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412170000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412180000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412190000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412200000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412210000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412220000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412230000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412240000/verif.nc +Would remove temporary output output/data/baselines/ICON-CH1-EPS/202412250000/verif.nc + +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408140000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408140000.log + jobid: 1041 + reason: Missing output files: output/data/runs/b30a-0209f019/202408140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408140000.ok + wildcards: run_id=b30a-0209f019, init_time=202408140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410090000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410090000.log + jobid: 1209 + reason: Missing output files: output/data/runs/b30a-0209f019/202410090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410090000.ok + wildcards: run_id=b30a-0209f019, init_time=202410090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404240000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404240000.log + jobid: 705 + reason: Missing output files: output/data/runs/b30a-0209f019/202404240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404240000.ok + wildcards: run_id=b30a-0209f019, init_time=202404240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401030000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401030000.log + jobid: 369 + reason: Missing output files: output/data/runs/b30a-0209f019/202401030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401030000.ok + wildcards: run_id=b30a-0209f019, init_time=202401030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406190000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406190000.log + jobid: 873 + reason: Missing output files: output/data/runs/b30a-0209f019/202406190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406190000.ok + wildcards: run_id=b30a-0209f019, init_time=202406190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412040000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412040000.log + jobid: 1377 + reason: Missing output files: output/data/runs/b30a-0209f019/202412040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412040000.ok + wildcards: run_id=b30a-0209f019, init_time=202412040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412050000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412050000.log + jobid: 1380 + reason: Missing output files: output/data/runs/b30a-0209f019/202412050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412050000.ok + wildcards: run_id=b30a-0209f019, init_time=202412050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410100000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410100000.log + jobid: 1212 + reason: Missing output files: output/data/runs/b30a-0209f019/202410100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410100000.ok + wildcards: run_id=b30a-0209f019, init_time=202410100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404250000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404250000.log + jobid: 708 + reason: Missing output files: output/data/runs/b30a-0209f019/202404250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404250000.ok + wildcards: run_id=b30a-0209f019, init_time=202404250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406200000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406200000.log + jobid: 876 + reason: Missing output files: output/data/runs/b30a-0209f019/202406200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406200000.ok + wildcards: run_id=b30a-0209f019, init_time=202406200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401040000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401040000.log + jobid: 372 + reason: Missing output files: output/data/runs/b30a-0209f019/202401040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401040000.ok + wildcards: run_id=b30a-0209f019, init_time=202401040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408150000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408150000.log + jobid: 1044 + reason: Missing output files: output/data/runs/b30a-0209f019/202408150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408150000.ok + wildcards: run_id=b30a-0209f019, init_time=202408150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408160000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408160000.log + jobid: 1047 + reason: Missing output files: output/data/runs/b30a-0209f019/202408160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408160000.ok + wildcards: run_id=b30a-0209f019, init_time=202408160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401050000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401050000.log + jobid: 375 + reason: Missing output files: output/data/runs/b30a-0209f019/202401050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401050000.ok + wildcards: run_id=b30a-0209f019, init_time=202401050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404260000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404260000.log + jobid: 711 + reason: Missing output files: output/data/runs/b30a-0209f019/202404260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404260000.ok + wildcards: run_id=b30a-0209f019, init_time=202404260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410110000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410110000.log + jobid: 1215 + reason: Missing output files: output/data/runs/b30a-0209f019/202410110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410110000.ok + wildcards: run_id=b30a-0209f019, init_time=202410110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412060000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412060000.log + jobid: 1383 + reason: Missing output files: output/data/runs/b30a-0209f019/202412060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412060000.ok + wildcards: run_id=b30a-0209f019, init_time=202412060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406210000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406210000.log + jobid: 879 + reason: Missing output files: output/data/runs/b30a-0209f019/202406210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406210000.ok + wildcards: run_id=b30a-0209f019, init_time=202406210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406220000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406220000.log + jobid: 882 + reason: Missing output files: output/data/runs/b30a-0209f019/202406220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406220000.ok + wildcards: run_id=b30a-0209f019, init_time=202406220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412070000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412070000.log + jobid: 1386 + reason: Missing output files: output/data/runs/b30a-0209f019/202412070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412070000.ok + wildcards: run_id=b30a-0209f019, init_time=202412070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410120000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410120000.log + jobid: 1218 + reason: Missing output files: output/data/runs/b30a-0209f019/202410120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410120000.ok + wildcards: run_id=b30a-0209f019, init_time=202410120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401060000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401060000.log + jobid: 378 + reason: Missing output files: output/data/runs/b30a-0209f019/202401060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401060000.ok + wildcards: run_id=b30a-0209f019, init_time=202401060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404270000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404270000.log + jobid: 714 + reason: Missing output files: output/data/runs/b30a-0209f019/202404270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404270000.ok + wildcards: run_id=b30a-0209f019, init_time=202404270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408170000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408170000.log + jobid: 1050 + reason: Missing output files: output/data/runs/b30a-0209f019/202408170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408170000.ok + wildcards: run_id=b30a-0209f019, init_time=202408170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408180000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408180000.log + jobid: 1053 + reason: Missing output files: output/data/runs/b30a-0209f019/202408180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408180000.ok + wildcards: run_id=b30a-0209f019, init_time=202408180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404280000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404280000.log + jobid: 717 + reason: Missing output files: output/data/runs/b30a-0209f019/202404280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404280000.ok + wildcards: run_id=b30a-0209f019, init_time=202404280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410130000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410130000.log + jobid: 1221 + reason: Missing output files: output/data/runs/b30a-0209f019/202410130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410130000.ok + wildcards: run_id=b30a-0209f019, init_time=202410130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401070000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401070000.log + jobid: 381 + reason: Missing output files: output/data/runs/b30a-0209f019/202401070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401070000.ok + wildcards: run_id=b30a-0209f019, init_time=202401070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412080000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412080000.log + jobid: 1389 + reason: Missing output files: output/data/runs/b30a-0209f019/202412080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412080000.ok + wildcards: run_id=b30a-0209f019, init_time=202412080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406230000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406230000.log + jobid: 885 + reason: Missing output files: output/data/runs/b30a-0209f019/202406230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406230000.ok + wildcards: run_id=b30a-0209f019, init_time=202406230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406240000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406240000.log + jobid: 888 + reason: Missing output files: output/data/runs/b30a-0209f019/202406240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406240000.ok + wildcards: run_id=b30a-0209f019, init_time=202406240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412090000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412090000.log + jobid: 1392 + reason: Missing output files: output/data/runs/b30a-0209f019/202412090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412090000.ok + wildcards: run_id=b30a-0209f019, init_time=202412090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401080000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401080000.log + jobid: 384 + reason: Missing output files: output/data/runs/b30a-0209f019/202401080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401080000.ok + wildcards: run_id=b30a-0209f019, init_time=202401080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410140000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410140000.log + jobid: 1224 + reason: Missing output files: output/data/runs/b30a-0209f019/202410140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410140000.ok + wildcards: run_id=b30a-0209f019, init_time=202410140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404290000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404290000.log + jobid: 720 + reason: Missing output files: output/data/runs/b30a-0209f019/202404290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404290000.ok + wildcards: run_id=b30a-0209f019, init_time=202404290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408190000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408190000.log + jobid: 1056 + reason: Missing output files: output/data/runs/b30a-0209f019/202408190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408190000.ok + wildcards: run_id=b30a-0209f019, init_time=202408190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408200000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408200000.log + jobid: 1059 + reason: Missing output files: output/data/runs/b30a-0209f019/202408200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408200000.ok + wildcards: run_id=b30a-0209f019, init_time=202408200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404300000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404300000.log + jobid: 723 + reason: Missing output files: output/data/runs/b30a-0209f019/202404300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404300000.ok + wildcards: run_id=b30a-0209f019, init_time=202404300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410150000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410150000.log + jobid: 1227 + reason: Missing output files: output/data/runs/b30a-0209f019/202410150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410150000.ok + wildcards: run_id=b30a-0209f019, init_time=202410150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401090000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401090000.log + jobid: 387 + reason: Missing output files: output/data/runs/b30a-0209f019/202401090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401090000.ok + wildcards: run_id=b30a-0209f019, init_time=202401090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412100000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412100000.log + jobid: 1395 + reason: Missing output files: output/data/runs/b30a-0209f019/202412100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412100000.ok + wildcards: run_id=b30a-0209f019, init_time=202412100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406250000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406250000.log + jobid: 891 + reason: Missing output files: output/data/runs/b30a-0209f019/202406250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406250000.ok + wildcards: run_id=b30a-0209f019, init_time=202406250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406260000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406260000.log + jobid: 894 + reason: Missing output files: output/data/runs/b30a-0209f019/202406260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406260000.ok + wildcards: run_id=b30a-0209f019, init_time=202406260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412110000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412110000.log + jobid: 1398 + reason: Missing output files: output/data/runs/b30a-0209f019/202412110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412110000.ok + wildcards: run_id=b30a-0209f019, init_time=202412110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401100000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401100000.log + jobid: 390 + reason: Missing output files: output/data/runs/b30a-0209f019/202401100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401100000.ok + wildcards: run_id=b30a-0209f019, init_time=202401100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410160000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410160000.log + jobid: 1230 + reason: Missing output files: output/data/runs/b30a-0209f019/202410160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410160000.ok + wildcards: run_id=b30a-0209f019, init_time=202410160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408210000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408210000.log + jobid: 1062 + reason: Missing output files: output/data/runs/b30a-0209f019/202408210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408210000.ok + wildcards: run_id=b30a-0209f019, init_time=202408210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405010000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405010000.log + jobid: 726 + reason: Missing output files: output/data/runs/b30a-0209f019/202405010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405010000.ok + wildcards: run_id=b30a-0209f019, init_time=202405010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408220000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408220000.log + jobid: 1065 + reason: Missing output files: output/data/runs/b30a-0209f019/202408220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408220000.ok + wildcards: run_id=b30a-0209f019, init_time=202408220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405020000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405020000.log + jobid: 729 + reason: Missing output files: output/data/runs/b30a-0209f019/202405020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405020000.ok + wildcards: run_id=b30a-0209f019, init_time=202405020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410170000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410170000.log + jobid: 1233 + reason: Missing output files: output/data/runs/b30a-0209f019/202410170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410170000.ok + wildcards: run_id=b30a-0209f019, init_time=202410170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401110000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401110000.log + jobid: 393 + reason: Missing output files: output/data/runs/b30a-0209f019/202401110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401110000.ok + wildcards: run_id=b30a-0209f019, init_time=202401110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412120000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412120000.log + jobid: 1401 + reason: Missing output files: output/data/runs/b30a-0209f019/202412120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412120000.ok + wildcards: run_id=b30a-0209f019, init_time=202412120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406270000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406270000.log + jobid: 897 + reason: Missing output files: output/data/runs/b30a-0209f019/202406270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406270000.ok + wildcards: run_id=b30a-0209f019, init_time=202406270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406280000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406280000.log + jobid: 900 + reason: Missing output files: output/data/runs/b30a-0209f019/202406280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406280000.ok + wildcards: run_id=b30a-0209f019, init_time=202406280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412130000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412130000.log + jobid: 1404 + reason: Missing output files: output/data/runs/b30a-0209f019/202412130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412130000.ok + wildcards: run_id=b30a-0209f019, init_time=202412130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405030000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405030000.log + jobid: 732 + reason: Missing output files: output/data/runs/b30a-0209f019/202405030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405030000.ok + wildcards: run_id=b30a-0209f019, init_time=202405030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408230000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408230000.log + jobid: 1068 + reason: Missing output files: output/data/runs/b30a-0209f019/202408230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408230000.ok + wildcards: run_id=b30a-0209f019, init_time=202408230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410180000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410180000.log + jobid: 1236 + reason: Missing output files: output/data/runs/b30a-0209f019/202410180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410180000.ok + wildcards: run_id=b30a-0209f019, init_time=202410180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401120000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401120000.log + jobid: 396 + reason: Missing output files: output/data/runs/b30a-0209f019/202401120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401120000.ok + wildcards: run_id=b30a-0209f019, init_time=202401120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412140000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412140000.log + jobid: 1407 + reason: Missing output files: output/data/runs/b30a-0209f019/202412140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412140000.ok + wildcards: run_id=b30a-0209f019, init_time=202412140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406290000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406290000.log + jobid: 903 + reason: Missing output files: output/data/runs/b30a-0209f019/202406290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406290000.ok + wildcards: run_id=b30a-0209f019, init_time=202406290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401130000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401130000.log + jobid: 399 + reason: Missing output files: output/data/runs/b30a-0209f019/202401130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401130000.ok + wildcards: run_id=b30a-0209f019, init_time=202401130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408240000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408240000.log + jobid: 1071 + reason: Missing output files: output/data/runs/b30a-0209f019/202408240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408240000.ok + wildcards: run_id=b30a-0209f019, init_time=202408240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405040000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405040000.log + jobid: 735 + reason: Missing output files: output/data/runs/b30a-0209f019/202405040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405040000.ok + wildcards: run_id=b30a-0209f019, init_time=202405040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410190000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410190000.log + jobid: 1239 + reason: Missing output files: output/data/runs/b30a-0209f019/202410190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410190000.ok + wildcards: run_id=b30a-0209f019, init_time=202410190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401140000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401140000.log + jobid: 402 + reason: Missing output files: output/data/runs/b30a-0209f019/202401140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401140000.ok + wildcards: run_id=b30a-0209f019, init_time=202401140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405050000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405050000.log + jobid: 738 + reason: Missing output files: output/data/runs/b30a-0209f019/202405050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405050000.ok + wildcards: run_id=b30a-0209f019, init_time=202405050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406300000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406300000.log + jobid: 906 + reason: Missing output files: output/data/runs/b30a-0209f019/202406300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406300000.ok + wildcards: run_id=b30a-0209f019, init_time=202406300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412150000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412150000.log + jobid: 1410 + reason: Missing output files: output/data/runs/b30a-0209f019/202412150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412150000.ok + wildcards: run_id=b30a-0209f019, init_time=202412150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410200000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410200000.log + jobid: 1242 + reason: Missing output files: output/data/runs/b30a-0209f019/202410200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410200000.ok + wildcards: run_id=b30a-0209f019, init_time=202410200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408250000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408250000.log + jobid: 1074 + reason: Missing output files: output/data/runs/b30a-0209f019/202408250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408250000.ok + wildcards: run_id=b30a-0209f019, init_time=202408250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412160000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412160000.log + jobid: 1413 + reason: Missing output files: output/data/runs/b30a-0209f019/202412160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412160000.ok + wildcards: run_id=b30a-0209f019, init_time=202412160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410210000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410210000.log + jobid: 1245 + reason: Missing output files: output/data/runs/b30a-0209f019/202410210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410210000.ok + wildcards: run_id=b30a-0209f019, init_time=202410210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408260000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408260000.log + jobid: 1077 + reason: Missing output files: output/data/runs/b30a-0209f019/202408260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408260000.ok + wildcards: run_id=b30a-0209f019, init_time=202408260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405060000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405060000.log + jobid: 741 + reason: Missing output files: output/data/runs/b30a-0209f019/202405060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405060000.ok + wildcards: run_id=b30a-0209f019, init_time=202405060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401150000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401150000.log + jobid: 405 + reason: Missing output files: output/data/runs/b30a-0209f019/202401150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401150000.ok + wildcards: run_id=b30a-0209f019, init_time=202401150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407010000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407010000.log + jobid: 909 + reason: Missing output files: output/data/runs/b30a-0209f019/202407010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407010000.ok + wildcards: run_id=b30a-0209f019, init_time=202407010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407020000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407020000.log + jobid: 912 + reason: Missing output files: output/data/runs/b30a-0209f019/202407020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407020000.ok + wildcards: run_id=b30a-0209f019, init_time=202407020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412170000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412170000.log + jobid: 1416 + reason: Missing output files: output/data/runs/b30a-0209f019/202412170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412170000.ok + wildcards: run_id=b30a-0209f019, init_time=202412170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401160000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401160000.log + jobid: 408 + reason: Missing output files: output/data/runs/b30a-0209f019/202401160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401160000.ok + wildcards: run_id=b30a-0209f019, init_time=202401160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405070000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405070000.log + jobid: 744 + reason: Missing output files: output/data/runs/b30a-0209f019/202405070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405070000.ok + wildcards: run_id=b30a-0209f019, init_time=202405070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408270000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408270000.log + jobid: 1080 + reason: Missing output files: output/data/runs/b30a-0209f019/202408270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408270000.ok + wildcards: run_id=b30a-0209f019, init_time=202408270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410220000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410220000.log + jobid: 1248 + reason: Missing output files: output/data/runs/b30a-0209f019/202410220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410220000.ok + wildcards: run_id=b30a-0209f019, init_time=202410220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410230000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410230000.log + jobid: 1251 + reason: Missing output files: output/data/runs/b30a-0209f019/202410230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410230000.ok + wildcards: run_id=b30a-0209f019, init_time=202410230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405080000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405080000.log + jobid: 747 + reason: Missing output files: output/data/runs/b30a-0209f019/202405080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405080000.ok + wildcards: run_id=b30a-0209f019, init_time=202405080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401170000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401170000.log + jobid: 411 + reason: Missing output files: output/data/runs/b30a-0209f019/202401170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401170000.ok + wildcards: run_id=b30a-0209f019, init_time=202401170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408280000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408280000.log + jobid: 1083 + reason: Missing output files: output/data/runs/b30a-0209f019/202408280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408280000.ok + wildcards: run_id=b30a-0209f019, init_time=202408280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407030000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407030000.log + jobid: 915 + reason: Missing output files: output/data/runs/b30a-0209f019/202407030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407030000.ok + wildcards: run_id=b30a-0209f019, init_time=202407030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412180000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412180000.log + jobid: 1419 + reason: Missing output files: output/data/runs/b30a-0209f019/202412180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412180000.ok + wildcards: run_id=b30a-0209f019, init_time=202412180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407040000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407040000.log + jobid: 918 + reason: Missing output files: output/data/runs/b30a-0209f019/202407040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407040000.ok + wildcards: run_id=b30a-0209f019, init_time=202407040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412190000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412190000.log + jobid: 1422 + reason: Missing output files: output/data/runs/b30a-0209f019/202412190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412190000.ok + wildcards: run_id=b30a-0209f019, init_time=202412190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408290000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408290000.log + jobid: 1086 + reason: Missing output files: output/data/runs/b30a-0209f019/202408290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408290000.ok + wildcards: run_id=b30a-0209f019, init_time=202408290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401180000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401180000.log + jobid: 414 + reason: Missing output files: output/data/runs/b30a-0209f019/202401180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401180000.ok + wildcards: run_id=b30a-0209f019, init_time=202401180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405090000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405090000.log + jobid: 750 + reason: Missing output files: output/data/runs/b30a-0209f019/202405090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405090000.ok + wildcards: run_id=b30a-0209f019, init_time=202405090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410240000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410240000.log + jobid: 1254 + reason: Missing output files: output/data/runs/b30a-0209f019/202410240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410240000.ok + wildcards: run_id=b30a-0209f019, init_time=202410240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401190000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401190000.log + jobid: 417 + reason: Missing output files: output/data/runs/b30a-0209f019/202401190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401190000.ok + wildcards: run_id=b30a-0209f019, init_time=202401190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407050000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407050000.log + jobid: 921 + reason: Missing output files: output/data/runs/b30a-0209f019/202407050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407050000.ok + wildcards: run_id=b30a-0209f019, init_time=202407050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408300000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408300000.log + jobid: 1089 + reason: Missing output files: output/data/runs/b30a-0209f019/202408300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408300000.ok + wildcards: run_id=b30a-0209f019, init_time=202408300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410250000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410250000.log + jobid: 1257 + reason: Missing output files: output/data/runs/b30a-0209f019/202410250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410250000.ok + wildcards: run_id=b30a-0209f019, init_time=202410250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405100000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405100000.log + jobid: 753 + reason: Missing output files: output/data/runs/b30a-0209f019/202405100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405100000.ok + wildcards: run_id=b30a-0209f019, init_time=202405100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412200000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412200000.log + jobid: 1425 + reason: Missing output files: output/data/runs/b30a-0209f019/202412200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412200000.ok + wildcards: run_id=b30a-0209f019, init_time=202412200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412210000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412210000.log + jobid: 1428 + reason: Missing output files: output/data/runs/b30a-0209f019/202412210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412210000.ok + wildcards: run_id=b30a-0209f019, init_time=202412210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405110000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405110000.log + jobid: 756 + reason: Missing output files: output/data/runs/b30a-0209f019/202405110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405110000.ok + wildcards: run_id=b30a-0209f019, init_time=202405110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410260000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410260000.log + jobid: 1260 + reason: Missing output files: output/data/runs/b30a-0209f019/202410260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410260000.ok + wildcards: run_id=b30a-0209f019, init_time=202410260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407060000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407060000.log + jobid: 924 + reason: Missing output files: output/data/runs/b30a-0209f019/202407060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407060000.ok + wildcards: run_id=b30a-0209f019, init_time=202407060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401200000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401200000.log + jobid: 420 + reason: Missing output files: output/data/runs/b30a-0209f019/202401200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401200000.ok + wildcards: run_id=b30a-0209f019, init_time=202401200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408310000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408310000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408310000.log + jobid: 1092 + reason: Missing output files: output/data/runs/b30a-0209f019/202408310000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408310000.ok + wildcards: run_id=b30a-0209f019, init_time=202408310000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401210000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401210000.log + jobid: 423 + reason: Missing output files: output/data/runs/b30a-0209f019/202401210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401210000.ok + wildcards: run_id=b30a-0209f019, init_time=202401210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407070000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407070000.log + jobid: 927 + reason: Missing output files: output/data/runs/b30a-0209f019/202407070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407070000.ok + wildcards: run_id=b30a-0209f019, init_time=202407070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409010000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409010000.log + jobid: 1095 + reason: Missing output files: output/data/runs/b30a-0209f019/202409010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409010000.ok + wildcards: run_id=b30a-0209f019, init_time=202409010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410270000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410270000.log + jobid: 1263 + reason: Missing output files: output/data/runs/b30a-0209f019/202410270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410270000.ok + wildcards: run_id=b30a-0209f019, init_time=202410270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412220000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412220000.log + jobid: 1431 + reason: Missing output files: output/data/runs/b30a-0209f019/202412220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412220000.ok + wildcards: run_id=b30a-0209f019, init_time=202412220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405120000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405120000.log + jobid: 759 + reason: Missing output files: output/data/runs/b30a-0209f019/202405120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405120000.ok + wildcards: run_id=b30a-0209f019, init_time=202405120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412230000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412230000.log + jobid: 1434 + reason: Missing output files: output/data/runs/b30a-0209f019/202412230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412230000.ok + wildcards: run_id=b30a-0209f019, init_time=202412230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405130000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405130000.log + jobid: 762 + reason: Missing output files: output/data/runs/b30a-0209f019/202405130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405130000.ok + wildcards: run_id=b30a-0209f019, init_time=202405130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps_baseline: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_all_6.png + jobid: 1455 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_all_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=centraleurope, season=all, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410280000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410280000.log + jobid: 1266 + reason: Missing output files: output/data/runs/b30a-0209f019/202410280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410280000.ok + wildcards: run_id=b30a-0209f019, init_time=202410280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409020000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409020000.log + jobid: 1098 + reason: Missing output files: output/data/runs/b30a-0209f019/202409020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409020000.ok + wildcards: run_id=b30a-0209f019, init_time=202409020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407080000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407080000.log + jobid: 930 + reason: Missing output files: output/data/runs/b30a-0209f019/202407080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407080000.ok + wildcards: run_id=b30a-0209f019, init_time=202407080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401220000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401220000.log + jobid: 426 + reason: Missing output files: output/data/runs/b30a-0209f019/202401220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401220000.ok + wildcards: run_id=b30a-0209f019, init_time=202401220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401230000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401230000.log + jobid: 429 + reason: Missing output files: output/data/runs/b30a-0209f019/202401230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401230000.ok + wildcards: run_id=b30a-0209f019, init_time=202401230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407090000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407090000.log + jobid: 933 + reason: Missing output files: output/data/runs/b30a-0209f019/202407090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407090000.ok + wildcards: run_id=b30a-0209f019, init_time=202407090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409030000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409030000.log + jobid: 1101 + reason: Missing output files: output/data/runs/b30a-0209f019/202409030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409030000.ok + wildcards: run_id=b30a-0209f019, init_time=202409030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410290000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410290000.log + jobid: 1269 + reason: Missing output files: output/data/runs/b30a-0209f019/202410290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410290000.ok + wildcards: run_id=b30a-0209f019, init_time=202410290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps_baseline: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_DJF_6.png + jobid: 1456 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_DJF_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=centraleurope, season=DJF, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412240000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412240000.log + jobid: 1437 + reason: Missing output files: output/data/runs/b30a-0209f019/202412240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412240000.ok + wildcards: run_id=b30a-0209f019, init_time=202412240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405140000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405140000.log + jobid: 765 + reason: Missing output files: output/data/runs/b30a-0209f019/202405140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405140000.ok + wildcards: run_id=b30a-0209f019, init_time=202405140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412250000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412250000.log + jobid: 1440 + reason: Missing output files: output/data/runs/b30a-0209f019/202412250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412250000.ok + wildcards: run_id=b30a-0209f019, init_time=202412250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405150000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405150000.log + jobid: 768 + reason: Missing output files: output/data/runs/b30a-0209f019/202405150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405150000.ok + wildcards: run_id=b30a-0209f019, init_time=202405150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps_baseline: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_MAM_6.png + jobid: 1457 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_MAM_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=centraleurope, season=MAM, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410300000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410300000.log + jobid: 1272 + reason: Missing output files: output/data/runs/b30a-0209f019/202410300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410300000.ok + wildcards: run_id=b30a-0209f019, init_time=202410300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409040000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409040000.log + jobid: 1104 + reason: Missing output files: output/data/runs/b30a-0209f019/202409040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409040000.ok + wildcards: run_id=b30a-0209f019, init_time=202409040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407100000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407100000.log + jobid: 936 + reason: Missing output files: output/data/runs/b30a-0209f019/202407100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407100000.ok + wildcards: run_id=b30a-0209f019, init_time=202407100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401240000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401240000.log + jobid: 432 + reason: Missing output files: output/data/runs/b30a-0209f019/202401240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401240000.ok + wildcards: run_id=b30a-0209f019, init_time=202401240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401250000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401250000.log + jobid: 435 + reason: Missing output files: output/data/runs/b30a-0209f019/202401250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401250000.ok + wildcards: run_id=b30a-0209f019, init_time=202401250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407110000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407110000.log + jobid: 939 + reason: Missing output files: output/data/runs/b30a-0209f019/202407110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407110000.ok + wildcards: run_id=b30a-0209f019, init_time=202407110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409050000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409050000.log + jobid: 1107 + reason: Missing output files: output/data/runs/b30a-0209f019/202409050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409050000.ok + wildcards: run_id=b30a-0209f019, init_time=202409050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410310000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410310000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410310000.log + jobid: 1275 + reason: Missing output files: output/data/runs/b30a-0209f019/202410310000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410310000.ok + wildcards: run_id=b30a-0209f019, init_time=202410310000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps_baseline: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_JJA_6.png + jobid: 1458 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_JJA_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=centraleurope, season=JJA, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405160000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405160000.log + jobid: 771 + reason: Missing output files: output/data/runs/b30a-0209f019/202405160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405160000.ok + wildcards: run_id=b30a-0209f019, init_time=202405160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405170000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405170000.log + jobid: 774 + reason: Missing output files: output/data/runs/b30a-0209f019/202405170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405170000.ok + wildcards: run_id=b30a-0209f019, init_time=202405170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps_baseline: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_SON_6.png + jobid: 1459 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_SON_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=centraleurope, season=SON, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409060000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409060000.log + jobid: 1110 + reason: Missing output files: output/data/runs/b30a-0209f019/202409060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409060000.ok + wildcards: run_id=b30a-0209f019, init_time=202409060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411010000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411010000.log + jobid: 1278 + reason: Missing output files: output/data/runs/b30a-0209f019/202411010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411010000.ok + wildcards: run_id=b30a-0209f019, init_time=202411010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407120000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407120000.log + jobid: 942 + reason: Missing output files: output/data/runs/b30a-0209f019/202407120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407120000.ok + wildcards: run_id=b30a-0209f019, init_time=202407120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401260000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401260000.log + jobid: 438 + reason: Missing output files: output/data/runs/b30a-0209f019/202401260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401260000.ok + wildcards: run_id=b30a-0209f019, init_time=202401260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407130000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407130000.log + jobid: 945 + reason: Missing output files: output/data/runs/b30a-0209f019/202407130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407130000.ok + wildcards: run_id=b30a-0209f019, init_time=202407130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409070000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409070000.log + jobid: 1113 + reason: Missing output files: output/data/runs/b30a-0209f019/202409070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409070000.ok + wildcards: run_id=b30a-0209f019, init_time=202409070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401270000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401270000.log + jobid: 441 + reason: Missing output files: output/data/runs/b30a-0209f019/202401270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401270000.ok + wildcards: run_id=b30a-0209f019, init_time=202401270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411020000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411020000.log + jobid: 1281 + reason: Missing output files: output/data/runs/b30a-0209f019/202411020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411020000.ok + wildcards: run_id=b30a-0209f019, init_time=202411020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps_baseline: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_all_6.png + jobid: 1460 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_all_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=switzerland, season=all, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405180000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405180000.log + jobid: 777 + reason: Missing output files: output/data/runs/b30a-0209f019/202405180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405180000.ok + wildcards: run_id=b30a-0209f019, init_time=202405180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405190000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405190000.log + jobid: 780 + reason: Missing output files: output/data/runs/b30a-0209f019/202405190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405190000.ok + wildcards: run_id=b30a-0209f019, init_time=202405190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps_baseline: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_DJF_6.png + jobid: 1461 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_DJF_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=switzerland, season=DJF, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411030000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411030000.log + jobid: 1284 + reason: Missing output files: output/data/runs/b30a-0209f019/202411030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411030000.ok + wildcards: run_id=b30a-0209f019, init_time=202411030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407140000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407140000.log + jobid: 948 + reason: Missing output files: output/data/runs/b30a-0209f019/202407140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407140000.ok + wildcards: run_id=b30a-0209f019, init_time=202407140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409080000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409080000.log + jobid: 1116 + reason: Missing output files: output/data/runs/b30a-0209f019/202409080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409080000.ok + wildcards: run_id=b30a-0209f019, init_time=202409080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401280000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401280000.log + jobid: 444 + reason: Missing output files: output/data/runs/b30a-0209f019/202401280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401280000.ok + wildcards: run_id=b30a-0209f019, init_time=202401280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401290000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401290000.log + jobid: 447 + reason: Missing output files: output/data/runs/b30a-0209f019/202401290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401290000.ok + wildcards: run_id=b30a-0209f019, init_time=202401290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409090000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409090000.log + jobid: 1119 + reason: Missing output files: output/data/runs/b30a-0209f019/202409090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409090000.ok + wildcards: run_id=b30a-0209f019, init_time=202409090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407150000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407150000.log + jobid: 951 + reason: Missing output files: output/data/runs/b30a-0209f019/202407150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407150000.ok + wildcards: run_id=b30a-0209f019, init_time=202407150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411040000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411040000.log + jobid: 1287 + reason: Missing output files: output/data/runs/b30a-0209f019/202411040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411040000.ok + wildcards: run_id=b30a-0209f019, init_time=202411040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps_baseline: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_MAM_6.png + jobid: 1462 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_MAM_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=switzerland, season=MAM, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405200000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405200000.log + jobid: 783 + reason: Missing output files: output/data/runs/b30a-0209f019/202405200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405200000.ok + wildcards: run_id=b30a-0209f019, init_time=202405200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405210000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405210000.log + jobid: 786 + reason: Missing output files: output/data/runs/b30a-0209f019/202405210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405210000.ok + wildcards: run_id=b30a-0209f019, init_time=202405210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps_baseline: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_JJA_6.png + jobid: 1463 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_JJA_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=switzerland, season=JJA, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411050000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411050000.log + jobid: 1290 + reason: Missing output files: output/data/runs/b30a-0209f019/202411050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411050000.ok + wildcards: run_id=b30a-0209f019, init_time=202411050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407160000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407160000.log + jobid: 954 + reason: Missing output files: output/data/runs/b30a-0209f019/202407160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407160000.ok + wildcards: run_id=b30a-0209f019, init_time=202407160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409100000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409100000.log + jobid: 1122 + reason: Missing output files: output/data/runs/b30a-0209f019/202409100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409100000.ok + wildcards: run_id=b30a-0209f019, init_time=202409100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401300000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401300000.log + jobid: 450 + reason: Missing output files: output/data/runs/b30a-0209f019/202401300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401300000.ok + wildcards: run_id=b30a-0209f019, init_time=202401300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401310000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401310000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401310000.log + jobid: 453 + reason: Missing output files: output/data/runs/b30a-0209f019/202401310000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401310000.ok + wildcards: run_id=b30a-0209f019, init_time=202401310000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps_baseline: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_SON_6.png + jobid: 1464 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_SON_6.png; Input files updated by another job: output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + wildcards: experiment=6b9203cd, baseline_id=ICON-CH1-EPS, param=SP_10M, metric=BIAS, region=switzerland, season=SON, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411060000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411060000.log + jobid: 1293 + reason: Missing output files: output/data/runs/b30a-0209f019/202411060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411060000.ok + wildcards: run_id=b30a-0209f019, init_time=202411060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409110000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409110000.log + jobid: 1125 + reason: Missing output files: output/data/runs/b30a-0209f019/202409110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409110000.ok + wildcards: run_id=b30a-0209f019, init_time=202409110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405220000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405220000.log + jobid: 789 + reason: Missing output files: output/data/runs/b30a-0209f019/202405220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405220000.ok + wildcards: run_id=b30a-0209f019, init_time=202405220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407170000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407170000.log + jobid: 957 + reason: Missing output files: output/data/runs/b30a-0209f019/202407170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407170000.ok + wildcards: run_id=b30a-0209f019, init_time=202407170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405230000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405230000.log + jobid: 792 + reason: Missing output files: output/data/runs/b30a-0209f019/202405230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405230000.ok + wildcards: run_id=b30a-0209f019, init_time=202405230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411070000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411070000.log + jobid: 1296 + reason: Missing output files: output/data/runs/b30a-0209f019/202411070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411070000.ok + wildcards: run_id=b30a-0209f019, init_time=202411070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407180000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407180000.log + jobid: 960 + reason: Missing output files: output/data/runs/b30a-0209f019/202407180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407180000.ok + wildcards: run_id=b30a-0209f019, init_time=202407180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409120000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409120000.log + jobid: 1128 + reason: Missing output files: output/data/runs/b30a-0209f019/202409120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409120000.ok + wildcards: run_id=b30a-0209f019, init_time=202409120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402010000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402010000.log + jobid: 456 + reason: Missing output files: output/data/runs/b30a-0209f019/202402010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402010000.ok + wildcards: run_id=b30a-0209f019, init_time=202402010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409130000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409130000.log + jobid: 1131 + reason: Missing output files: output/data/runs/b30a-0209f019/202409130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409130000.ok + wildcards: run_id=b30a-0209f019, init_time=202409130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402020000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402020000.log + jobid: 459 + reason: Missing output files: output/data/runs/b30a-0209f019/202402020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402020000.ok + wildcards: run_id=b30a-0209f019, init_time=202402020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411080000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411080000.log + jobid: 1299 + reason: Missing output files: output/data/runs/b30a-0209f019/202411080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411080000.ok + wildcards: run_id=b30a-0209f019, init_time=202411080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407190000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407190000.log + jobid: 963 + reason: Missing output files: output/data/runs/b30a-0209f019/202407190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407190000.ok + wildcards: run_id=b30a-0209f019, init_time=202407190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405240000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405240000.log + jobid: 795 + reason: Missing output files: output/data/runs/b30a-0209f019/202405240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405240000.ok + wildcards: run_id=b30a-0209f019, init_time=202405240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405250000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405250000.log + jobid: 798 + reason: Missing output files: output/data/runs/b30a-0209f019/202405250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405250000.ok + wildcards: run_id=b30a-0209f019, init_time=202405250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411090000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411090000.log + jobid: 1302 + reason: Missing output files: output/data/runs/b30a-0209f019/202411090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411090000.ok + wildcards: run_id=b30a-0209f019, init_time=202411090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407200000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407200000.log + jobid: 966 + reason: Missing output files: output/data/runs/b30a-0209f019/202407200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407200000.ok + wildcards: run_id=b30a-0209f019, init_time=202407200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402030000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402030000.log + jobid: 462 + reason: Missing output files: output/data/runs/b30a-0209f019/202402030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402030000.ok + wildcards: run_id=b30a-0209f019, init_time=202402030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409140000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409140000.log + jobid: 1134 + reason: Missing output files: output/data/runs/b30a-0209f019/202409140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409140000.ok + wildcards: run_id=b30a-0209f019, init_time=202409140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409150000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409150000.log + jobid: 1137 + reason: Missing output files: output/data/runs/b30a-0209f019/202409150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409150000.ok + wildcards: run_id=b30a-0209f019, init_time=202409150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402040000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402040000.log + jobid: 465 + reason: Missing output files: output/data/runs/b30a-0209f019/202402040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402040000.ok + wildcards: run_id=b30a-0209f019, init_time=202402040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407210000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407210000.log + jobid: 969 + reason: Missing output files: output/data/runs/b30a-0209f019/202407210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407210000.ok + wildcards: run_id=b30a-0209f019, init_time=202407210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411100000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411100000.log + jobid: 1305 + reason: Missing output files: output/data/runs/b30a-0209f019/202411100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411100000.ok + wildcards: run_id=b30a-0209f019, init_time=202411100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405260000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405260000.log + jobid: 801 + reason: Missing output files: output/data/runs/b30a-0209f019/202405260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405260000.ok + wildcards: run_id=b30a-0209f019, init_time=202405260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405270000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405270000.log + jobid: 804 + reason: Missing output files: output/data/runs/b30a-0209f019/202405270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405270000.ok + wildcards: run_id=b30a-0209f019, init_time=202405270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407220000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407220000.log + jobid: 972 + reason: Missing output files: output/data/runs/b30a-0209f019/202407220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407220000.ok + wildcards: run_id=b30a-0209f019, init_time=202407220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402050000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402050000.log + jobid: 468 + reason: Missing output files: output/data/runs/b30a-0209f019/202402050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402050000.ok + wildcards: run_id=b30a-0209f019, init_time=202402050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409160000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409160000.log + jobid: 1140 + reason: Missing output files: output/data/runs/b30a-0209f019/202409160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409160000.ok + wildcards: run_id=b30a-0209f019, init_time=202409160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411110000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411110000.log + jobid: 1308 + reason: Missing output files: output/data/runs/b30a-0209f019/202411110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411110000.ok + wildcards: run_id=b30a-0209f019, init_time=202411110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402060000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402060000.log + jobid: 471 + reason: Missing output files: output/data/runs/b30a-0209f019/202402060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402060000.ok + wildcards: run_id=b30a-0209f019, init_time=202402060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409170000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409170000.log + jobid: 1143 + reason: Missing output files: output/data/runs/b30a-0209f019/202409170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409170000.ok + wildcards: run_id=b30a-0209f019, init_time=202409170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407230000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407230000.log + jobid: 975 + reason: Missing output files: output/data/runs/b30a-0209f019/202407230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407230000.ok + wildcards: run_id=b30a-0209f019, init_time=202407230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411120000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411120000.log + jobid: 1311 + reason: Missing output files: output/data/runs/b30a-0209f019/202411120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411120000.ok + wildcards: run_id=b30a-0209f019, init_time=202411120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405280000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405280000.log + jobid: 807 + reason: Missing output files: output/data/runs/b30a-0209f019/202405280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405280000.ok + wildcards: run_id=b30a-0209f019, init_time=202405280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405290000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405290000.log + jobid: 810 + reason: Missing output files: output/data/runs/b30a-0209f019/202405290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405290000.ok + wildcards: run_id=b30a-0209f019, init_time=202405290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411130000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411130000.log + jobid: 1314 + reason: Missing output files: output/data/runs/b30a-0209f019/202411130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411130000.ok + wildcards: run_id=b30a-0209f019, init_time=202411130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407240000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407240000.log + jobid: 978 + reason: Missing output files: output/data/runs/b30a-0209f019/202407240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407240000.ok + wildcards: run_id=b30a-0209f019, init_time=202407240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409180000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409180000.log + jobid: 1146 + reason: Missing output files: output/data/runs/b30a-0209f019/202409180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409180000.ok + wildcards: run_id=b30a-0209f019, init_time=202409180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402070000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402070000.log + jobid: 474 + reason: Missing output files: output/data/runs/b30a-0209f019/202402070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402070000.ok + wildcards: run_id=b30a-0209f019, init_time=202402070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402080000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402080000.log + jobid: 477 + reason: Missing output files: output/data/runs/b30a-0209f019/202402080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402080000.ok + wildcards: run_id=b30a-0209f019, init_time=202402080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409190000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409190000.log + jobid: 1149 + reason: Missing output files: output/data/runs/b30a-0209f019/202409190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409190000.ok + wildcards: run_id=b30a-0209f019, init_time=202409190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407250000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407250000.log + jobid: 981 + reason: Missing output files: output/data/runs/b30a-0209f019/202407250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407250000.ok + wildcards: run_id=b30a-0209f019, init_time=202407250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411140000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411140000.log + jobid: 1317 + reason: Missing output files: output/data/runs/b30a-0209f019/202411140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411140000.ok + wildcards: run_id=b30a-0209f019, init_time=202411140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405300000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405300000.log + jobid: 813 + reason: Missing output files: output/data/runs/b30a-0209f019/202405300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405300000.ok + wildcards: run_id=b30a-0209f019, init_time=202405300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202405310000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202405310000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202405310000.log + jobid: 816 + reason: Missing output files: output/data/runs/b30a-0209f019/202405310000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202405310000.ok + wildcards: run_id=b30a-0209f019, init_time=202405310000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409200000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409200000.log + jobid: 1152 + reason: Missing output files: output/data/runs/b30a-0209f019/202409200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409200000.ok + wildcards: run_id=b30a-0209f019, init_time=202409200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402090000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402090000.log + jobid: 480 + reason: Missing output files: output/data/runs/b30a-0209f019/202402090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402090000.ok + wildcards: run_id=b30a-0209f019, init_time=202402090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407260000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407260000.log + jobid: 984 + reason: Missing output files: output/data/runs/b30a-0209f019/202407260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407260000.ok + wildcards: run_id=b30a-0209f019, init_time=202407260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411150000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411150000.log + jobid: 1320 + reason: Missing output files: output/data/runs/b30a-0209f019/202411150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411150000.ok + wildcards: run_id=b30a-0209f019, init_time=202411150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402100000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402100000.log + jobid: 483 + reason: Missing output files: output/data/runs/b30a-0209f019/202402100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402100000.ok + wildcards: run_id=b30a-0209f019, init_time=202402100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409210000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409210000.log + jobid: 1155 + reason: Missing output files: output/data/runs/b30a-0209f019/202409210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409210000.ok + wildcards: run_id=b30a-0209f019, init_time=202409210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407270000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407270000.log + jobid: 987 + reason: Missing output files: output/data/runs/b30a-0209f019/202407270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407270000.ok + wildcards: run_id=b30a-0209f019, init_time=202407270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411160000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411160000.log + jobid: 1323 + reason: Missing output files: output/data/runs/b30a-0209f019/202411160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411160000.ok + wildcards: run_id=b30a-0209f019, init_time=202411160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406010000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406010000.log + jobid: 819 + reason: Missing output files: output/data/runs/b30a-0209f019/202406010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406010000.ok + wildcards: run_id=b30a-0209f019, init_time=202406010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406020000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406020000.log + jobid: 822 + reason: Missing output files: output/data/runs/b30a-0209f019/202406020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406020000.ok + wildcards: run_id=b30a-0209f019, init_time=202406020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411170000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411170000.log + jobid: 1326 + reason: Missing output files: output/data/runs/b30a-0209f019/202411170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411170000.ok + wildcards: run_id=b30a-0209f019, init_time=202411170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407280000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407280000.log + jobid: 990 + reason: Missing output files: output/data/runs/b30a-0209f019/202407280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407280000.ok + wildcards: run_id=b30a-0209f019, init_time=202407280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409220000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409220000.log + jobid: 1158 + reason: Missing output files: output/data/runs/b30a-0209f019/202409220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409220000.ok + wildcards: run_id=b30a-0209f019, init_time=202409220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402110000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402110000.log + jobid: 486 + reason: Missing output files: output/data/runs/b30a-0209f019/202402110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402110000.ok + wildcards: run_id=b30a-0209f019, init_time=202402110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402120000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402120000.log + jobid: 489 + reason: Missing output files: output/data/runs/b30a-0209f019/202402120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402120000.ok + wildcards: run_id=b30a-0209f019, init_time=202402120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409230000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409230000.log + jobid: 1161 + reason: Missing output files: output/data/runs/b30a-0209f019/202409230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409230000.ok + wildcards: run_id=b30a-0209f019, init_time=202409230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407290000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407290000.log + jobid: 993 + reason: Missing output files: output/data/runs/b30a-0209f019/202407290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407290000.ok + wildcards: run_id=b30a-0209f019, init_time=202407290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411180000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411180000.log + jobid: 1329 + reason: Missing output files: output/data/runs/b30a-0209f019/202411180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411180000.ok + wildcards: run_id=b30a-0209f019, init_time=202411180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406030000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406030000.log + jobid: 825 + reason: Missing output files: output/data/runs/b30a-0209f019/202406030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406030000.ok + wildcards: run_id=b30a-0209f019, init_time=202406030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406040000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406040000.log + jobid: 828 + reason: Missing output files: output/data/runs/b30a-0209f019/202406040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406040000.ok + wildcards: run_id=b30a-0209f019, init_time=202406040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407300000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407300000.log + jobid: 996 + reason: Missing output files: output/data/runs/b30a-0209f019/202407300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407300000.ok + wildcards: run_id=b30a-0209f019, init_time=202407300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402130000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402130000.log + jobid: 492 + reason: Missing output files: output/data/runs/b30a-0209f019/202402130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402130000.ok + wildcards: run_id=b30a-0209f019, init_time=202402130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409240000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409240000.log + jobid: 1164 + reason: Missing output files: output/data/runs/b30a-0209f019/202409240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409240000.ok + wildcards: run_id=b30a-0209f019, init_time=202409240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411190000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411190000.log + jobid: 1332 + reason: Missing output files: output/data/runs/b30a-0209f019/202411190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411190000.ok + wildcards: run_id=b30a-0209f019, init_time=202411190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411200000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411200000.log + jobid: 1335 + reason: Missing output files: output/data/runs/b30a-0209f019/202411200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411200000.ok + wildcards: run_id=b30a-0209f019, init_time=202411200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402140000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402140000.log + jobid: 495 + reason: Missing output files: output/data/runs/b30a-0209f019/202402140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402140000.ok + wildcards: run_id=b30a-0209f019, init_time=202402140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202407310000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202407310000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202407310000.log + jobid: 999 + reason: Missing output files: output/data/runs/b30a-0209f019/202407310000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202407310000.ok + wildcards: run_id=b30a-0209f019, init_time=202407310000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409250000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409250000.log + jobid: 1167 + reason: Missing output files: output/data/runs/b30a-0209f019/202409250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409250000.ok + wildcards: run_id=b30a-0209f019, init_time=202409250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406050000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406050000.log + jobid: 831 + reason: Missing output files: output/data/runs/b30a-0209f019/202406050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406050000.ok + wildcards: run_id=b30a-0209f019, init_time=202406050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406060000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406060000.log + jobid: 834 + reason: Missing output files: output/data/runs/b30a-0209f019/202406060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406060000.ok + wildcards: run_id=b30a-0209f019, init_time=202406060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408010000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408010000.log + jobid: 1002 + reason: Missing output files: output/data/runs/b30a-0209f019/202408010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408010000.ok + wildcards: run_id=b30a-0209f019, init_time=202408010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402150000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402150000.log + jobid: 498 + reason: Missing output files: output/data/runs/b30a-0209f019/202402150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402150000.ok + wildcards: run_id=b30a-0209f019, init_time=202402150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411210000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411210000.log + jobid: 1338 + reason: Missing output files: output/data/runs/b30a-0209f019/202411210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411210000.ok + wildcards: run_id=b30a-0209f019, init_time=202411210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409260000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409260000.log + jobid: 1170 + reason: Missing output files: output/data/runs/b30a-0209f019/202409260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409260000.ok + wildcards: run_id=b30a-0209f019, init_time=202409260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408020000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408020000.log + jobid: 1005 + reason: Missing output files: output/data/runs/b30a-0209f019/202408020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408020000.ok + wildcards: run_id=b30a-0209f019, init_time=202408020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402160000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402160000.log + jobid: 501 + reason: Missing output files: output/data/runs/b30a-0209f019/202402160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402160000.ok + wildcards: run_id=b30a-0209f019, init_time=202402160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409270000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409270000.log + jobid: 1173 + reason: Missing output files: output/data/runs/b30a-0209f019/202409270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409270000.ok + wildcards: run_id=b30a-0209f019, init_time=202409270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411220000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411220000.log + jobid: 1341 + reason: Missing output files: output/data/runs/b30a-0209f019/202411220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411220000.ok + wildcards: run_id=b30a-0209f019, init_time=202411220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406070000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406070000.log + jobid: 837 + reason: Missing output files: output/data/runs/b30a-0209f019/202406070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406070000.ok + wildcards: run_id=b30a-0209f019, init_time=202406070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406080000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406080000.log + jobid: 840 + reason: Missing output files: output/data/runs/b30a-0209f019/202406080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406080000.ok + wildcards: run_id=b30a-0209f019, init_time=202406080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411230000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411230000.log + jobid: 1344 + reason: Missing output files: output/data/runs/b30a-0209f019/202411230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411230000.ok + wildcards: run_id=b30a-0209f019, init_time=202411230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409280000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409280000.log + jobid: 1176 + reason: Missing output files: output/data/runs/b30a-0209f019/202409280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409280000.ok + wildcards: run_id=b30a-0209f019, init_time=202409280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408030000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408030000.log + jobid: 1008 + reason: Missing output files: output/data/runs/b30a-0209f019/202408030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408030000.ok + wildcards: run_id=b30a-0209f019, init_time=202408030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402170000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402170000.log + jobid: 504 + reason: Missing output files: output/data/runs/b30a-0209f019/202402170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402170000.ok + wildcards: run_id=b30a-0209f019, init_time=202402170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402180000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402180000.log + jobid: 507 + reason: Missing output files: output/data/runs/b30a-0209f019/202402180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402180000.ok + wildcards: run_id=b30a-0209f019, init_time=202402180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408040000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408040000.log + jobid: 1011 + reason: Missing output files: output/data/runs/b30a-0209f019/202408040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408040000.ok + wildcards: run_id=b30a-0209f019, init_time=202408040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409290000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409290000.log + jobid: 1179 + reason: Missing output files: output/data/runs/b30a-0209f019/202409290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409290000.ok + wildcards: run_id=b30a-0209f019, init_time=202409290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411240000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411240000.log + jobid: 1347 + reason: Missing output files: output/data/runs/b30a-0209f019/202411240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411240000.ok + wildcards: run_id=b30a-0209f019, init_time=202411240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406090000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406090000.log + jobid: 843 + reason: Missing output files: output/data/runs/b30a-0209f019/202406090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406090000.ok + wildcards: run_id=b30a-0209f019, init_time=202406090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406100000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406100000.log + jobid: 846 + reason: Missing output files: output/data/runs/b30a-0209f019/202406100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406100000.ok + wildcards: run_id=b30a-0209f019, init_time=202406100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411250000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411250000.log + jobid: 1350 + reason: Missing output files: output/data/runs/b30a-0209f019/202411250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411250000.ok + wildcards: run_id=b30a-0209f019, init_time=202411250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202409300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202409300000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202409300000.log + jobid: 1182 + reason: Missing output files: output/data/runs/b30a-0209f019/202409300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202409300000.ok + wildcards: run_id=b30a-0209f019, init_time=202409300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408050000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408050000.log + jobid: 1014 + reason: Missing output files: output/data/runs/b30a-0209f019/202408050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408050000.ok + wildcards: run_id=b30a-0209f019, init_time=202408050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402190000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402190000.log + jobid: 510 + reason: Missing output files: output/data/runs/b30a-0209f019/202402190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402190000.ok + wildcards: run_id=b30a-0209f019, init_time=202402190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402200000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402200000.log + jobid: 513 + reason: Missing output files: output/data/runs/b30a-0209f019/202402200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402200000.ok + wildcards: run_id=b30a-0209f019, init_time=202402200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408060000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408060000.log + jobid: 1017 + reason: Missing output files: output/data/runs/b30a-0209f019/202408060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408060000.ok + wildcards: run_id=b30a-0209f019, init_time=202408060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410010000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410010000.log + jobid: 1185 + reason: Missing output files: output/data/runs/b30a-0209f019/202410010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410010000.ok + wildcards: run_id=b30a-0209f019, init_time=202410010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411260000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411260000.log + jobid: 1353 + reason: Missing output files: output/data/runs/b30a-0209f019/202411260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411260000.ok + wildcards: run_id=b30a-0209f019, init_time=202411260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406110000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406110000.log + jobid: 849 + reason: Missing output files: output/data/runs/b30a-0209f019/202406110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406110000.ok + wildcards: run_id=b30a-0209f019, init_time=202406110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406120000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406120000.log + jobid: 852 + reason: Missing output files: output/data/runs/b30a-0209f019/202406120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406120000.ok + wildcards: run_id=b30a-0209f019, init_time=202406120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411270000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411270000.log + jobid: 1356 + reason: Missing output files: output/data/runs/b30a-0209f019/202411270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411270000.ok + wildcards: run_id=b30a-0209f019, init_time=202411270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410020000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410020000.log + jobid: 1188 + reason: Missing output files: output/data/runs/b30a-0209f019/202410020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410020000.ok + wildcards: run_id=b30a-0209f019, init_time=202410020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408070000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408070000.log + jobid: 1020 + reason: Missing output files: output/data/runs/b30a-0209f019/202408070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408070000.ok + wildcards: run_id=b30a-0209f019, init_time=202408070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402210000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402210000.log + jobid: 516 + reason: Missing output files: output/data/runs/b30a-0209f019/202402210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402210000.ok + wildcards: run_id=b30a-0209f019, init_time=202402210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402220000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402220000.log + jobid: 519 + reason: Missing output files: output/data/runs/b30a-0209f019/202402220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402220000.ok + wildcards: run_id=b30a-0209f019, init_time=202402220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408080000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408080000.log + jobid: 1023 + reason: Missing output files: output/data/runs/b30a-0209f019/202408080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408080000.ok + wildcards: run_id=b30a-0209f019, init_time=202408080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410030000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410030000.log + jobid: 1191 + reason: Missing output files: output/data/runs/b30a-0209f019/202410030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410030000.ok + wildcards: run_id=b30a-0209f019, init_time=202410030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406130000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406130000.log + jobid: 855 + reason: Missing output files: output/data/runs/b30a-0209f019/202406130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406130000.ok + wildcards: run_id=b30a-0209f019, init_time=202406130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411280000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411280000.log + jobid: 1359 + reason: Missing output files: output/data/runs/b30a-0209f019/202411280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411280000.ok + wildcards: run_id=b30a-0209f019, init_time=202411280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406140000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406140000.log + jobid: 858 + reason: Missing output files: output/data/runs/b30a-0209f019/202406140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406140000.ok + wildcards: run_id=b30a-0209f019, init_time=202406140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411290000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411290000.log + jobid: 1362 + reason: Missing output files: output/data/runs/b30a-0209f019/202411290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411290000.ok + wildcards: run_id=b30a-0209f019, init_time=202411290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408090000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408090000.log + jobid: 1026 + reason: Missing output files: output/data/runs/b30a-0209f019/202408090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408090000.ok + wildcards: run_id=b30a-0209f019, init_time=202408090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410040000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410040000.log + jobid: 1194 + reason: Missing output files: output/data/runs/b30a-0209f019/202410040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410040000.ok + wildcards: run_id=b30a-0209f019, init_time=202410040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402230000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402230000.log + jobid: 522 + reason: Missing output files: output/data/runs/b30a-0209f019/202402230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402230000.ok + wildcards: run_id=b30a-0209f019, init_time=202402230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408100000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408100000.log + jobid: 1029 + reason: Missing output files: output/data/runs/b30a-0209f019/202408100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408100000.ok + wildcards: run_id=b30a-0209f019, init_time=202408100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402240000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402240000.log + jobid: 525 + reason: Missing output files: output/data/runs/b30a-0209f019/202402240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402240000.ok + wildcards: run_id=b30a-0209f019, init_time=202402240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410050000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410050000.log + jobid: 1197 + reason: Missing output files: output/data/runs/b30a-0209f019/202410050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410050000.ok + wildcards: run_id=b30a-0209f019, init_time=202410050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406150000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406150000.log + jobid: 861 + reason: Missing output files: output/data/runs/b30a-0209f019/202406150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406150000.ok + wildcards: run_id=b30a-0209f019, init_time=202406150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202411300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202411300000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202411300000.log + jobid: 1365 + reason: Missing output files: output/data/runs/b30a-0209f019/202411300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202411300000.ok + wildcards: run_id=b30a-0209f019, init_time=202411300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412010000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412010000.log + jobid: 1368 + reason: Missing output files: output/data/runs/b30a-0209f019/202412010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412010000.ok + wildcards: run_id=b30a-0209f019, init_time=202412010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406160000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406160000.log + jobid: 864 + reason: Missing output files: output/data/runs/b30a-0209f019/202406160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406160000.ok + wildcards: run_id=b30a-0209f019, init_time=202406160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410060000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410060000.log + jobid: 1200 + reason: Missing output files: output/data/runs/b30a-0209f019/202410060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410060000.ok + wildcards: run_id=b30a-0209f019, init_time=202410060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408110000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408110000.log + jobid: 1032 + reason: Missing output files: output/data/runs/b30a-0209f019/202408110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408110000.ok + wildcards: run_id=b30a-0209f019, init_time=202408110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402250000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402250000.log + jobid: 528 + reason: Missing output files: output/data/runs/b30a-0209f019/202402250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402250000.ok + wildcards: run_id=b30a-0209f019, init_time=202402250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408120000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408120000.log + jobid: 1035 + reason: Missing output files: output/data/runs/b30a-0209f019/202408120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408120000.ok + wildcards: run_id=b30a-0209f019, init_time=202408120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402260000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402260000.log + jobid: 531 + reason: Missing output files: output/data/runs/b30a-0209f019/202402260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402260000.ok + wildcards: run_id=b30a-0209f019, init_time=202402260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410070000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410070000.log + jobid: 1203 + reason: Missing output files: output/data/runs/b30a-0209f019/202410070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410070000.ok + wildcards: run_id=b30a-0209f019, init_time=202410070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412020000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412020000.log + jobid: 1371 + reason: Missing output files: output/data/runs/b30a-0209f019/202412020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412020000.ok + wildcards: run_id=b30a-0209f019, init_time=202412020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406170000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406170000.log + jobid: 867 + reason: Missing output files: output/data/runs/b30a-0209f019/202406170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406170000.ok + wildcards: run_id=b30a-0209f019, init_time=202406170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202412030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202412030000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202412030000.log + jobid: 1374 + reason: Missing output files: output/data/runs/b30a-0209f019/202412030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202412030000.ok + wildcards: run_id=b30a-0209f019, init_time=202412030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202406180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202406180000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202406180000.log + jobid: 870 + reason: Missing output files: output/data/runs/b30a-0209f019/202406180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202406180000.ok + wildcards: run_id=b30a-0209f019, init_time=202406180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202410080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202410080000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202410080000.log + jobid: 1206 + reason: Missing output files: output/data/runs/b30a-0209f019/202410080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202410080000.ok + wildcards: run_id=b30a-0209f019, init_time=202410080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402270000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402270000.log + jobid: 534 + reason: Missing output files: output/data/runs/b30a-0209f019/202402270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402270000.ok + wildcards: run_id=b30a-0209f019, init_time=202402270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202408130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202408130000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202408130000.log + jobid: 1038 + reason: Missing output files: output/data/runs/b30a-0209f019/202408130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202408130000.ok + wildcards: run_id=b30a-0209f019, init_time=202408130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402280000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402280000.log + jobid: 537 + reason: Missing output files: output/data/runs/b30a-0209f019/202402280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402280000.ok + wildcards: run_id=b30a-0209f019, init_time=202402280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202402290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202402290000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202402290000.log + jobid: 540 + reason: Missing output files: output/data/runs/b30a-0209f019/202402290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202402290000.ok + wildcards: run_id=b30a-0209f019, init_time=202402290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403010000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403010000.log + jobid: 543 + reason: Missing output files: output/data/runs/b30a-0209f019/202403010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403010000.ok + wildcards: run_id=b30a-0209f019, init_time=202403010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403020000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403020000.log + jobid: 546 + reason: Missing output files: output/data/runs/b30a-0209f019/202403020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403020000.ok + wildcards: run_id=b30a-0209f019, init_time=202403020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403030000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403030000.log + jobid: 549 + reason: Missing output files: output/data/runs/b30a-0209f019/202403030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403030000.ok + wildcards: run_id=b30a-0209f019, init_time=202403030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403040000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403040000.log + jobid: 552 + reason: Missing output files: output/data/runs/b30a-0209f019/202403040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403040000.ok + wildcards: run_id=b30a-0209f019, init_time=202403040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403050000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403050000.log + jobid: 555 + reason: Missing output files: output/data/runs/b30a-0209f019/202403050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403050000.ok + wildcards: run_id=b30a-0209f019, init_time=202403050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403060000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403060000.log + jobid: 558 + reason: Missing output files: output/data/runs/b30a-0209f019/202403060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403060000.ok + wildcards: run_id=b30a-0209f019, init_time=202403060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403070000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403070000.log + jobid: 561 + reason: Missing output files: output/data/runs/b30a-0209f019/202403070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403070000.ok + wildcards: run_id=b30a-0209f019, init_time=202403070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403080000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403080000.log + jobid: 564 + reason: Missing output files: output/data/runs/b30a-0209f019/202403080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403080000.ok + wildcards: run_id=b30a-0209f019, init_time=202403080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403090000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403090000.log + jobid: 567 + reason: Missing output files: output/data/runs/b30a-0209f019/202403090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403090000.ok + wildcards: run_id=b30a-0209f019, init_time=202403090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403100000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403100000.log + jobid: 570 + reason: Missing output files: output/data/runs/b30a-0209f019/202403100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403100000.ok + wildcards: run_id=b30a-0209f019, init_time=202403100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403110000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403110000.log + jobid: 573 + reason: Missing output files: output/data/runs/b30a-0209f019/202403110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403110000.ok + wildcards: run_id=b30a-0209f019, init_time=202403110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403120000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403120000.log + jobid: 576 + reason: Missing output files: output/data/runs/b30a-0209f019/202403120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403120000.ok + wildcards: run_id=b30a-0209f019, init_time=202403120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403130000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403130000.log + jobid: 579 + reason: Missing output files: output/data/runs/b30a-0209f019/202403130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403130000.ok + wildcards: run_id=b30a-0209f019, init_time=202403130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403140000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403140000.log + jobid: 582 + reason: Missing output files: output/data/runs/b30a-0209f019/202403140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403140000.ok + wildcards: run_id=b30a-0209f019, init_time=202403140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403150000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403150000.log + jobid: 585 + reason: Missing output files: output/data/runs/b30a-0209f019/202403150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403150000.ok + wildcards: run_id=b30a-0209f019, init_time=202403150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403160000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403160000.log + jobid: 588 + reason: Missing output files: output/data/runs/b30a-0209f019/202403160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403160000.ok + wildcards: run_id=b30a-0209f019, init_time=202403160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403170000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403170000.log + jobid: 591 + reason: Missing output files: output/data/runs/b30a-0209f019/202403170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403170000.ok + wildcards: run_id=b30a-0209f019, init_time=202403170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403180000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403180000.log + jobid: 594 + reason: Missing output files: output/data/runs/b30a-0209f019/202403180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403180000.ok + wildcards: run_id=b30a-0209f019, init_time=202403180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403190000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403190000.log + jobid: 597 + reason: Missing output files: output/data/runs/b30a-0209f019/202403190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403190000.ok + wildcards: run_id=b30a-0209f019, init_time=202403190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403200000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403200000.log + jobid: 600 + reason: Missing output files: output/data/runs/b30a-0209f019/202403200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403200000.ok + wildcards: run_id=b30a-0209f019, init_time=202403200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403210000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403210000.log + jobid: 603 + reason: Missing output files: output/data/runs/b30a-0209f019/202403210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403210000.ok + wildcards: run_id=b30a-0209f019, init_time=202403210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403220000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403220000.log + jobid: 606 + reason: Missing output files: output/data/runs/b30a-0209f019/202403220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403220000.ok + wildcards: run_id=b30a-0209f019, init_time=202403220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403230000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403230000.log + jobid: 609 + reason: Missing output files: output/data/runs/b30a-0209f019/202403230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403230000.ok + wildcards: run_id=b30a-0209f019, init_time=202403230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403240000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403240000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403240000.log + jobid: 612 + reason: Missing output files: output/data/runs/b30a-0209f019/202403240000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403240000.ok + wildcards: run_id=b30a-0209f019, init_time=202403240000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403250000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403250000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403250000.log + jobid: 615 + reason: Missing output files: output/data/runs/b30a-0209f019/202403250000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403250000.ok + wildcards: run_id=b30a-0209f019, init_time=202403250000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403260000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403260000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403260000.log + jobid: 618 + reason: Missing output files: output/data/runs/b30a-0209f019/202403260000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403260000.ok + wildcards: run_id=b30a-0209f019, init_time=202403260000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403270000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403270000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403270000.log + jobid: 621 + reason: Missing output files: output/data/runs/b30a-0209f019/202403270000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403270000.ok + wildcards: run_id=b30a-0209f019, init_time=202403270000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403280000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403280000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403280000.log + jobid: 624 + reason: Missing output files: output/data/runs/b30a-0209f019/202403280000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403280000.ok + wildcards: run_id=b30a-0209f019, init_time=202403280000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403290000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403290000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403290000.log + jobid: 627 + reason: Missing output files: output/data/runs/b30a-0209f019/202403290000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403290000.ok + wildcards: run_id=b30a-0209f019, init_time=202403290000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403300000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403300000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403300000.log + jobid: 630 + reason: Missing output files: output/data/runs/b30a-0209f019/202403300000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403300000.ok + wildcards: run_id=b30a-0209f019, init_time=202403300000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202403310000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202403310000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202403310000.log + jobid: 633 + reason: Missing output files: output/data/runs/b30a-0209f019/202403310000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202403310000.ok + wildcards: run_id=b30a-0209f019, init_time=202403310000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404010000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404010000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404010000.log + jobid: 636 + reason: Missing output files: output/data/runs/b30a-0209f019/202404010000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404010000.ok + wildcards: run_id=b30a-0209f019, init_time=202404010000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404020000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404020000.log + jobid: 639 + reason: Missing output files: output/data/runs/b30a-0209f019/202404020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404020000.ok + wildcards: run_id=b30a-0209f019, init_time=202404020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404030000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404030000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404030000.log + jobid: 642 + reason: Missing output files: output/data/runs/b30a-0209f019/202404030000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404030000.ok + wildcards: run_id=b30a-0209f019, init_time=202404030000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404040000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404040000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404040000.log + jobid: 645 + reason: Missing output files: output/data/runs/b30a-0209f019/202404040000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404040000.ok + wildcards: run_id=b30a-0209f019, init_time=202404040000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404050000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404050000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404050000.log + jobid: 648 + reason: Missing output files: output/data/runs/b30a-0209f019/202404050000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404050000.ok + wildcards: run_id=b30a-0209f019, init_time=202404050000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404060000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404060000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404060000.log + jobid: 651 + reason: Missing output files: output/data/runs/b30a-0209f019/202404060000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404060000.ok + wildcards: run_id=b30a-0209f019, init_time=202404060000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404070000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404070000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404070000.log + jobid: 654 + reason: Missing output files: output/data/runs/b30a-0209f019/202404070000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404070000.ok + wildcards: run_id=b30a-0209f019, init_time=202404070000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404080000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404080000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404080000.log + jobid: 657 + reason: Missing output files: output/data/runs/b30a-0209f019/202404080000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404080000.ok + wildcards: run_id=b30a-0209f019, init_time=202404080000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404090000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404090000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404090000.log + jobid: 660 + reason: Missing output files: output/data/runs/b30a-0209f019/202404090000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404090000.ok + wildcards: run_id=b30a-0209f019, init_time=202404090000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404100000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404100000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404100000.log + jobid: 663 + reason: Missing output files: output/data/runs/b30a-0209f019/202404100000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404100000.ok + wildcards: run_id=b30a-0209f019, init_time=202404100000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404110000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404110000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404110000.log + jobid: 666 + reason: Missing output files: output/data/runs/b30a-0209f019/202404110000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404110000.ok + wildcards: run_id=b30a-0209f019, init_time=202404110000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404120000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404120000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404120000.log + jobid: 669 + reason: Missing output files: output/data/runs/b30a-0209f019/202404120000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404120000.ok + wildcards: run_id=b30a-0209f019, init_time=202404120000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404130000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404130000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404130000.log + jobid: 672 + reason: Missing output files: output/data/runs/b30a-0209f019/202404130000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404130000.ok + wildcards: run_id=b30a-0209f019, init_time=202404130000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404140000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404140000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404140000.log + jobid: 675 + reason: Missing output files: output/data/runs/b30a-0209f019/202404140000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404140000.ok + wildcards: run_id=b30a-0209f019, init_time=202404140000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404150000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404150000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404150000.log + jobid: 678 + reason: Missing output files: output/data/runs/b30a-0209f019/202404150000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404150000.ok + wildcards: run_id=b30a-0209f019, init_time=202404150000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404160000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404160000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404160000.log + jobid: 681 + reason: Missing output files: output/data/runs/b30a-0209f019/202404160000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404160000.ok + wildcards: run_id=b30a-0209f019, init_time=202404160000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404170000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404170000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404170000.log + jobid: 684 + reason: Missing output files: output/data/runs/b30a-0209f019/202404170000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404170000.ok + wildcards: run_id=b30a-0209f019, init_time=202404170000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404180000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404180000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404180000.log + jobid: 687 + reason: Missing output files: output/data/runs/b30a-0209f019/202404180000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404180000.ok + wildcards: run_id=b30a-0209f019, init_time=202404180000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404190000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404190000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404190000.log + jobid: 690 + reason: Missing output files: output/data/runs/b30a-0209f019/202404190000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404190000.ok + wildcards: run_id=b30a-0209f019, init_time=202404190000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404200000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404200000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404200000.log + jobid: 693 + reason: Missing output files: output/data/runs/b30a-0209f019/202404200000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404200000.ok + wildcards: run_id=b30a-0209f019, init_time=202404200000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404210000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404210000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404210000.log + jobid: 696 + reason: Missing output files: output/data/runs/b30a-0209f019/202404210000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404210000.ok + wildcards: run_id=b30a-0209f019, init_time=202404210000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404220000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404220000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404220000.log + jobid: 699 + reason: Missing output files: output/data/runs/b30a-0209f019/202404220000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404220000.ok + wildcards: run_id=b30a-0209f019, init_time=202404220000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202401020000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202401020000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202401020000.log + jobid: 363 + reason: Missing output files: output/data/runs/b30a-0209f019/202401020000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202401020000.ok + wildcards: run_id=b30a-0209f019, init_time=202401020000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics: + input: src/verification/__init__.py, src/data_input/__init__.py, workflow/scripts/verif_single_init.py, output/logs/execute_inference/b30a-0209f019-202404230000.ok, /store_new/mch/msopr/ml/datasets/mch-realch1-fdb-1km-2005-2025-1h-pl13-v1.0.zarr + output: output/data/runs/b30a-0209f019/202404230000/verif.nc + log: output/logs/verif_metrics/b30a-0209f019-202404230000.log + jobid: 702 + reason: Missing output files: output/data/runs/b30a-0209f019/202404230000/verif.nc; Input files updated by another job: output/logs/execute_inference/b30a-0209f019-202404230000.ok + wildcards: run_id=b30a-0209f019, init_time=202404230000 + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=60, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_aggregation: + input: workflow/scripts/verif_aggregation.py, output/data/runs/b30a-0209f019/202401020000/verif.nc, output/data/runs/b30a-0209f019/202401030000/verif.nc, output/data/runs/b30a-0209f019/202401040000/verif.nc, output/data/runs/b30a-0209f019/202401050000/verif.nc, output/data/runs/b30a-0209f019/202401060000/verif.nc, output/data/runs/b30a-0209f019/202401070000/verif.nc, output/data/runs/b30a-0209f019/202401080000/verif.nc, output/data/runs/b30a-0209f019/202401090000/verif.nc, output/data/runs/b30a-0209f019/202401100000/verif.nc, output/data/runs/b30a-0209f019/202401110000/verif.nc, output/data/runs/b30a-0209f019/202401120000/verif.nc, output/data/runs/b30a-0209f019/202401130000/verif.nc, output/data/runs/b30a-0209f019/202401140000/verif.nc, output/data/runs/b30a-0209f019/202401150000/verif.nc, output/data/runs/b30a-0209f019/202401160000/verif.nc, output/data/runs/b30a-0209f019/202401170000/verif.nc, output/data/runs/b30a-0209f019/202401180000/verif.nc, output/data/runs/b30a-0209f019/202401190000/verif.nc, output/data/runs/b30a-0209f019/202401200000/verif.nc, output/data/runs/b30a-0209f019/202401210000/verif.nc, output/data/runs/b30a-0209f019/202401220000/verif.nc, output/data/runs/b30a-0209f019/202401230000/verif.nc, output/data/runs/b30a-0209f019/202401240000/verif.nc, output/data/runs/b30a-0209f019/202401250000/verif.nc, output/data/runs/b30a-0209f019/202401260000/verif.nc, output/data/runs/b30a-0209f019/202401270000/verif.nc, output/data/runs/b30a-0209f019/202401280000/verif.nc, output/data/runs/b30a-0209f019/202401290000/verif.nc, output/data/runs/b30a-0209f019/202401300000/verif.nc, output/data/runs/b30a-0209f019/202401310000/verif.nc, output/data/runs/b30a-0209f019/202402010000/verif.nc, output/data/runs/b30a-0209f019/202402020000/verif.nc, output/data/runs/b30a-0209f019/202402030000/verif.nc, output/data/runs/b30a-0209f019/202402040000/verif.nc, output/data/runs/b30a-0209f019/202402050000/verif.nc, output/data/runs/b30a-0209f019/202402060000/verif.nc, output/data/runs/b30a-0209f019/202402070000/verif.nc, output/data/runs/b30a-0209f019/202402080000/verif.nc, output/data/runs/b30a-0209f019/202402090000/verif.nc, output/data/runs/b30a-0209f019/202402100000/verif.nc, output/data/runs/b30a-0209f019/202402110000/verif.nc, output/data/runs/b30a-0209f019/202402120000/verif.nc, output/data/runs/b30a-0209f019/202402130000/verif.nc, output/data/runs/b30a-0209f019/202402140000/verif.nc, output/data/runs/b30a-0209f019/202402150000/verif.nc, output/data/runs/b30a-0209f019/202402160000/verif.nc, output/data/runs/b30a-0209f019/202402170000/verif.nc, output/data/runs/b30a-0209f019/202402180000/verif.nc, output/data/runs/b30a-0209f019/202402190000/verif.nc, output/data/runs/b30a-0209f019/202402200000/verif.nc, output/data/runs/b30a-0209f019/202402210000/verif.nc, output/data/runs/b30a-0209f019/202402220000/verif.nc, output/data/runs/b30a-0209f019/202402230000/verif.nc, output/data/runs/b30a-0209f019/202402240000/verif.nc, output/data/runs/b30a-0209f019/202402250000/verif.nc, output/data/runs/b30a-0209f019/202402260000/verif.nc, output/data/runs/b30a-0209f019/202402270000/verif.nc, output/data/runs/b30a-0209f019/202402280000/verif.nc, output/data/runs/b30a-0209f019/202402290000/verif.nc, output/data/runs/b30a-0209f019/202403010000/verif.nc, output/data/runs/b30a-0209f019/202403020000/verif.nc, output/data/runs/b30a-0209f019/202403030000/verif.nc, output/data/runs/b30a-0209f019/202403040000/verif.nc, output/data/runs/b30a-0209f019/202403050000/verif.nc, output/data/runs/b30a-0209f019/202403060000/verif.nc, output/data/runs/b30a-0209f019/202403070000/verif.nc, output/data/runs/b30a-0209f019/202403080000/verif.nc, output/data/runs/b30a-0209f019/202403090000/verif.nc, output/data/runs/b30a-0209f019/202403100000/verif.nc, output/data/runs/b30a-0209f019/202403110000/verif.nc, output/data/runs/b30a-0209f019/202403120000/verif.nc, output/data/runs/b30a-0209f019/202403130000/verif.nc, output/data/runs/b30a-0209f019/202403140000/verif.nc, output/data/runs/b30a-0209f019/202403150000/verif.nc, output/data/runs/b30a-0209f019/202403160000/verif.nc, output/data/runs/b30a-0209f019/202403170000/verif.nc, output/data/runs/b30a-0209f019/202403180000/verif.nc, output/data/runs/b30a-0209f019/202403190000/verif.nc, output/data/runs/b30a-0209f019/202403200000/verif.nc, output/data/runs/b30a-0209f019/202403210000/verif.nc, output/data/runs/b30a-0209f019/202403220000/verif.nc, output/data/runs/b30a-0209f019/202403230000/verif.nc, output/data/runs/b30a-0209f019/202403240000/verif.nc, output/data/runs/b30a-0209f019/202403250000/verif.nc, output/data/runs/b30a-0209f019/202403260000/verif.nc, output/data/runs/b30a-0209f019/202403270000/verif.nc, output/data/runs/b30a-0209f019/202403280000/verif.nc, output/data/runs/b30a-0209f019/202403290000/verif.nc, output/data/runs/b30a-0209f019/202403300000/verif.nc, output/data/runs/b30a-0209f019/202403310000/verif.nc, output/data/runs/b30a-0209f019/202404010000/verif.nc, output/data/runs/b30a-0209f019/202404020000/verif.nc, output/data/runs/b30a-0209f019/202404030000/verif.nc, output/data/runs/b30a-0209f019/202404040000/verif.nc, output/data/runs/b30a-0209f019/202404050000/verif.nc, output/data/runs/b30a-0209f019/202404060000/verif.nc, output/data/runs/b30a-0209f019/202404070000/verif.nc, output/data/runs/b30a-0209f019/202404080000/verif.nc, output/data/runs/b30a-0209f019/202404090000/verif.nc, output/data/runs/b30a-0209f019/202404100000/verif.nc, output/data/runs/b30a-0209f019/202404110000/verif.nc, output/data/runs/b30a-0209f019/202404120000/verif.nc, output/data/runs/b30a-0209f019/202404130000/verif.nc, output/data/runs/b30a-0209f019/202404140000/verif.nc, output/data/runs/b30a-0209f019/202404150000/verif.nc, output/data/runs/b30a-0209f019/202404160000/verif.nc, output/data/runs/b30a-0209f019/202404170000/verif.nc, output/data/runs/b30a-0209f019/202404180000/verif.nc, output/data/runs/b30a-0209f019/202404190000/verif.nc, output/data/runs/b30a-0209f019/202404200000/verif.nc, output/data/runs/b30a-0209f019/202404210000/verif.nc, output/data/runs/b30a-0209f019/202404220000/verif.nc, output/data/runs/b30a-0209f019/202404230000/verif.nc, output/data/runs/b30a-0209f019/202404240000/verif.nc, output/data/runs/b30a-0209f019/202404250000/verif.nc, output/data/runs/b30a-0209f019/202404260000/verif.nc, output/data/runs/b30a-0209f019/202404270000/verif.nc, output/data/runs/b30a-0209f019/202404280000/verif.nc, output/data/runs/b30a-0209f019/202404290000/verif.nc, output/data/runs/b30a-0209f019/202404300000/verif.nc, output/data/runs/b30a-0209f019/202405010000/verif.nc, output/data/runs/b30a-0209f019/202405020000/verif.nc, output/data/runs/b30a-0209f019/202405030000/verif.nc, output/data/runs/b30a-0209f019/202405040000/verif.nc, output/data/runs/b30a-0209f019/202405050000/verif.nc, output/data/runs/b30a-0209f019/202405060000/verif.nc, output/data/runs/b30a-0209f019/202405070000/verif.nc, output/data/runs/b30a-0209f019/202405080000/verif.nc, output/data/runs/b30a-0209f019/202405090000/verif.nc, output/data/runs/b30a-0209f019/202405100000/verif.nc, output/data/runs/b30a-0209f019/202405110000/verif.nc, output/data/runs/b30a-0209f019/202405120000/verif.nc, output/data/runs/b30a-0209f019/202405130000/verif.nc, output/data/runs/b30a-0209f019/202405140000/verif.nc, output/data/runs/b30a-0209f019/202405150000/verif.nc, output/data/runs/b30a-0209f019/202405160000/verif.nc, output/data/runs/b30a-0209f019/202405170000/verif.nc, output/data/runs/b30a-0209f019/202405180000/verif.nc, output/data/runs/b30a-0209f019/202405190000/verif.nc, output/data/runs/b30a-0209f019/202405200000/verif.nc, output/data/runs/b30a-0209f019/202405210000/verif.nc, output/data/runs/b30a-0209f019/202405220000/verif.nc, output/data/runs/b30a-0209f019/202405230000/verif.nc, output/data/runs/b30a-0209f019/202405240000/verif.nc, output/data/runs/b30a-0209f019/202405250000/verif.nc, output/data/runs/b30a-0209f019/202405260000/verif.nc, output/data/runs/b30a-0209f019/202405270000/verif.nc, output/data/runs/b30a-0209f019/202405280000/verif.nc, output/data/runs/b30a-0209f019/202405290000/verif.nc, output/data/runs/b30a-0209f019/202405300000/verif.nc, output/data/runs/b30a-0209f019/202405310000/verif.nc, output/data/runs/b30a-0209f019/202406010000/verif.nc, output/data/runs/b30a-0209f019/202406020000/verif.nc, output/data/runs/b30a-0209f019/202406030000/verif.nc, output/data/runs/b30a-0209f019/202406040000/verif.nc, output/data/runs/b30a-0209f019/202406050000/verif.nc, output/data/runs/b30a-0209f019/202406060000/verif.nc, output/data/runs/b30a-0209f019/202406070000/verif.nc, output/data/runs/b30a-0209f019/202406080000/verif.nc, output/data/runs/b30a-0209f019/202406090000/verif.nc, output/data/runs/b30a-0209f019/202406100000/verif.nc, output/data/runs/b30a-0209f019/202406110000/verif.nc, output/data/runs/b30a-0209f019/202406120000/verif.nc, output/data/runs/b30a-0209f019/202406130000/verif.nc, output/data/runs/b30a-0209f019/202406140000/verif.nc, output/data/runs/b30a-0209f019/202406150000/verif.nc, output/data/runs/b30a-0209f019/202406160000/verif.nc, output/data/runs/b30a-0209f019/202406170000/verif.nc, output/data/runs/b30a-0209f019/202406180000/verif.nc, output/data/runs/b30a-0209f019/202406190000/verif.nc, output/data/runs/b30a-0209f019/202406200000/verif.nc, output/data/runs/b30a-0209f019/202406210000/verif.nc, output/data/runs/b30a-0209f019/202406220000/verif.nc, output/data/runs/b30a-0209f019/202406230000/verif.nc, output/data/runs/b30a-0209f019/202406240000/verif.nc, output/data/runs/b30a-0209f019/202406250000/verif.nc, output/data/runs/b30a-0209f019/202406260000/verif.nc, output/data/runs/b30a-0209f019/202406270000/verif.nc, output/data/runs/b30a-0209f019/202406280000/verif.nc, output/data/runs/b30a-0209f019/202406290000/verif.nc, output/data/runs/b30a-0209f019/202406300000/verif.nc, output/data/runs/b30a-0209f019/202407010000/verif.nc, output/data/runs/b30a-0209f019/202407020000/verif.nc, output/data/runs/b30a-0209f019/202407030000/verif.nc, output/data/runs/b30a-0209f019/202407040000/verif.nc, output/data/runs/b30a-0209f019/202407050000/verif.nc, output/data/runs/b30a-0209f019/202407060000/verif.nc, output/data/runs/b30a-0209f019/202407070000/verif.nc, output/data/runs/b30a-0209f019/202407080000/verif.nc, output/data/runs/b30a-0209f019/202407090000/verif.nc, output/data/runs/b30a-0209f019/202407100000/verif.nc, output/data/runs/b30a-0209f019/202407110000/verif.nc, output/data/runs/b30a-0209f019/202407120000/verif.nc, output/data/runs/b30a-0209f019/202407130000/verif.nc, output/data/runs/b30a-0209f019/202407140000/verif.nc, output/data/runs/b30a-0209f019/202407150000/verif.nc, output/data/runs/b30a-0209f019/202407160000/verif.nc, output/data/runs/b30a-0209f019/202407170000/verif.nc, output/data/runs/b30a-0209f019/202407180000/verif.nc, output/data/runs/b30a-0209f019/202407190000/verif.nc, output/data/runs/b30a-0209f019/202407200000/verif.nc, output/data/runs/b30a-0209f019/202407210000/verif.nc, output/data/runs/b30a-0209f019/202407220000/verif.nc, output/data/runs/b30a-0209f019/202407230000/verif.nc, output/data/runs/b30a-0209f019/202407240000/verif.nc, output/data/runs/b30a-0209f019/202407250000/verif.nc, output/data/runs/b30a-0209f019/202407260000/verif.nc, output/data/runs/b30a-0209f019/202407270000/verif.nc, output/data/runs/b30a-0209f019/202407280000/verif.nc, output/data/runs/b30a-0209f019/202407290000/verif.nc, output/data/runs/b30a-0209f019/202407300000/verif.nc, output/data/runs/b30a-0209f019/202407310000/verif.nc, output/data/runs/b30a-0209f019/202408010000/verif.nc, output/data/runs/b30a-0209f019/202408020000/verif.nc, output/data/runs/b30a-0209f019/202408030000/verif.nc, output/data/runs/b30a-0209f019/202408040000/verif.nc, output/data/runs/b30a-0209f019/202408050000/verif.nc, output/data/runs/b30a-0209f019/202408060000/verif.nc, output/data/runs/b30a-0209f019/202408070000/verif.nc, output/data/runs/b30a-0209f019/202408080000/verif.nc, output/data/runs/b30a-0209f019/202408090000/verif.nc, output/data/runs/b30a-0209f019/202408100000/verif.nc, output/data/runs/b30a-0209f019/202408110000/verif.nc, output/data/runs/b30a-0209f019/202408120000/verif.nc, output/data/runs/b30a-0209f019/202408130000/verif.nc, output/data/runs/b30a-0209f019/202408140000/verif.nc, output/data/runs/b30a-0209f019/202408150000/verif.nc, output/data/runs/b30a-0209f019/202408160000/verif.nc, output/data/runs/b30a-0209f019/202408170000/verif.nc, output/data/runs/b30a-0209f019/202408180000/verif.nc, output/data/runs/b30a-0209f019/202408190000/verif.nc, output/data/runs/b30a-0209f019/202408200000/verif.nc, output/data/runs/b30a-0209f019/202408210000/verif.nc, output/data/runs/b30a-0209f019/202408220000/verif.nc, output/data/runs/b30a-0209f019/202408230000/verif.nc, output/data/runs/b30a-0209f019/202408240000/verif.nc, output/data/runs/b30a-0209f019/202408250000/verif.nc, output/data/runs/b30a-0209f019/202408260000/verif.nc, output/data/runs/b30a-0209f019/202408270000/verif.nc, output/data/runs/b30a-0209f019/202408280000/verif.nc, output/data/runs/b30a-0209f019/202408290000/verif.nc, output/data/runs/b30a-0209f019/202408300000/verif.nc, output/data/runs/b30a-0209f019/202408310000/verif.nc, output/data/runs/b30a-0209f019/202409010000/verif.nc, output/data/runs/b30a-0209f019/202409020000/verif.nc, output/data/runs/b30a-0209f019/202409030000/verif.nc, output/data/runs/b30a-0209f019/202409040000/verif.nc, output/data/runs/b30a-0209f019/202409050000/verif.nc, output/data/runs/b30a-0209f019/202409060000/verif.nc, output/data/runs/b30a-0209f019/202409070000/verif.nc, output/data/runs/b30a-0209f019/202409080000/verif.nc, output/data/runs/b30a-0209f019/202409090000/verif.nc, output/data/runs/b30a-0209f019/202409100000/verif.nc, output/data/runs/b30a-0209f019/202409110000/verif.nc, output/data/runs/b30a-0209f019/202409120000/verif.nc, output/data/runs/b30a-0209f019/202409130000/verif.nc, output/data/runs/b30a-0209f019/202409140000/verif.nc, output/data/runs/b30a-0209f019/202409150000/verif.nc, output/data/runs/b30a-0209f019/202409160000/verif.nc, output/data/runs/b30a-0209f019/202409170000/verif.nc, output/data/runs/b30a-0209f019/202409180000/verif.nc, output/data/runs/b30a-0209f019/202409190000/verif.nc, output/data/runs/b30a-0209f019/202409200000/verif.nc, output/data/runs/b30a-0209f019/202409210000/verif.nc, output/data/runs/b30a-0209f019/202409220000/verif.nc, output/data/runs/b30a-0209f019/202409230000/verif.nc, output/data/runs/b30a-0209f019/202409240000/verif.nc, output/data/runs/b30a-0209f019/202409250000/verif.nc, output/data/runs/b30a-0209f019/202409260000/verif.nc, output/data/runs/b30a-0209f019/202409270000/verif.nc, output/data/runs/b30a-0209f019/202409280000/verif.nc, output/data/runs/b30a-0209f019/202409290000/verif.nc, output/data/runs/b30a-0209f019/202409300000/verif.nc, output/data/runs/b30a-0209f019/202410010000/verif.nc, output/data/runs/b30a-0209f019/202410020000/verif.nc, output/data/runs/b30a-0209f019/202410030000/verif.nc, output/data/runs/b30a-0209f019/202410040000/verif.nc, output/data/runs/b30a-0209f019/202410050000/verif.nc, output/data/runs/b30a-0209f019/202410060000/verif.nc, output/data/runs/b30a-0209f019/202410070000/verif.nc, output/data/runs/b30a-0209f019/202410080000/verif.nc, output/data/runs/b30a-0209f019/202410090000/verif.nc, output/data/runs/b30a-0209f019/202410100000/verif.nc, output/data/runs/b30a-0209f019/202410110000/verif.nc, output/data/runs/b30a-0209f019/202410120000/verif.nc, output/data/runs/b30a-0209f019/202410130000/verif.nc, output/data/runs/b30a-0209f019/202410140000/verif.nc, output/data/runs/b30a-0209f019/202410150000/verif.nc, output/data/runs/b30a-0209f019/202410160000/verif.nc, output/data/runs/b30a-0209f019/202410170000/verif.nc, output/data/runs/b30a-0209f019/202410180000/verif.nc, output/data/runs/b30a-0209f019/202410190000/verif.nc, output/data/runs/b30a-0209f019/202410200000/verif.nc, output/data/runs/b30a-0209f019/202410210000/verif.nc, output/data/runs/b30a-0209f019/202410220000/verif.nc, output/data/runs/b30a-0209f019/202410230000/verif.nc, output/data/runs/b30a-0209f019/202410240000/verif.nc, output/data/runs/b30a-0209f019/202410250000/verif.nc, output/data/runs/b30a-0209f019/202410260000/verif.nc, output/data/runs/b30a-0209f019/202410270000/verif.nc, output/data/runs/b30a-0209f019/202410280000/verif.nc, output/data/runs/b30a-0209f019/202410290000/verif.nc, output/data/runs/b30a-0209f019/202410300000/verif.nc, output/data/runs/b30a-0209f019/202410310000/verif.nc, output/data/runs/b30a-0209f019/202411010000/verif.nc, output/data/runs/b30a-0209f019/202411020000/verif.nc, output/data/runs/b30a-0209f019/202411030000/verif.nc, output/data/runs/b30a-0209f019/202411040000/verif.nc, output/data/runs/b30a-0209f019/202411050000/verif.nc, output/data/runs/b30a-0209f019/202411060000/verif.nc, output/data/runs/b30a-0209f019/202411070000/verif.nc, output/data/runs/b30a-0209f019/202411080000/verif.nc, output/data/runs/b30a-0209f019/202411090000/verif.nc, output/data/runs/b30a-0209f019/202411100000/verif.nc, output/data/runs/b30a-0209f019/202411110000/verif.nc, output/data/runs/b30a-0209f019/202411120000/verif.nc, output/data/runs/b30a-0209f019/202411130000/verif.nc, output/data/runs/b30a-0209f019/202411140000/verif.nc, output/data/runs/b30a-0209f019/202411150000/verif.nc, output/data/runs/b30a-0209f019/202411160000/verif.nc, output/data/runs/b30a-0209f019/202411170000/verif.nc, output/data/runs/b30a-0209f019/202411180000/verif.nc, output/data/runs/b30a-0209f019/202411190000/verif.nc, output/data/runs/b30a-0209f019/202411200000/verif.nc, output/data/runs/b30a-0209f019/202411210000/verif.nc, output/data/runs/b30a-0209f019/202411220000/verif.nc, output/data/runs/b30a-0209f019/202411230000/verif.nc, output/data/runs/b30a-0209f019/202411240000/verif.nc, output/data/runs/b30a-0209f019/202411250000/verif.nc, output/data/runs/b30a-0209f019/202411260000/verif.nc, output/data/runs/b30a-0209f019/202411270000/verif.nc, output/data/runs/b30a-0209f019/202411280000/verif.nc, output/data/runs/b30a-0209f019/202411290000/verif.nc, output/data/runs/b30a-0209f019/202411300000/verif.nc, output/data/runs/b30a-0209f019/202412010000/verif.nc, output/data/runs/b30a-0209f019/202412020000/verif.nc, output/data/runs/b30a-0209f019/202412030000/verif.nc, output/data/runs/b30a-0209f019/202412040000/verif.nc, output/data/runs/b30a-0209f019/202412050000/verif.nc, output/data/runs/b30a-0209f019/202412060000/verif.nc, output/data/runs/b30a-0209f019/202412070000/verif.nc, output/data/runs/b30a-0209f019/202412080000/verif.nc, output/data/runs/b30a-0209f019/202412090000/verif.nc, output/data/runs/b30a-0209f019/202412100000/verif.nc, output/data/runs/b30a-0209f019/202412110000/verif.nc, output/data/runs/b30a-0209f019/202412120000/verif.nc, output/data/runs/b30a-0209f019/202412130000/verif.nc, output/data/runs/b30a-0209f019/202412140000/verif.nc, output/data/runs/b30a-0209f019/202412150000/verif.nc, output/data/runs/b30a-0209f019/202412160000/verif.nc, output/data/runs/b30a-0209f019/202412170000/verif.nc, output/data/runs/b30a-0209f019/202412180000/verif.nc, output/data/runs/b30a-0209f019/202412190000/verif.nc, output/data/runs/b30a-0209f019/202412200000/verif.nc, output/data/runs/b30a-0209f019/202412210000/verif.nc, output/data/runs/b30a-0209f019/202412220000/verif.nc, output/data/runs/b30a-0209f019/202412230000/verif.nc, output/data/runs/b30a-0209f019/202412240000/verif.nc, output/data/runs/b30a-0209f019/202412250000/verif.nc + output: output/data/runs/b30a-0209f019/verif_aggregated.nc + log: output/logs/verif_metrics_aggregation/b30a-0209f019.log + jobid: 362 + reason: Input files updated by another job: output/data/runs/b30a-0209f019/202407280000/verif.nc, output/data/runs/b30a-0209f019/202401040000/verif.nc, output/data/runs/b30a-0209f019/202406040000/verif.nc, output/data/runs/b30a-0209f019/202410170000/verif.nc, output/data/runs/b30a-0209f019/202407060000/verif.nc, output/data/runs/b30a-0209f019/202410260000/verif.nc, output/data/runs/b30a-0209f019/202408090000/verif.nc, output/data/runs/b30a-0209f019/202407230000/verif.nc, output/data/runs/b30a-0209f019/202408060000/verif.nc, output/data/runs/b30a-0209f019/202407180000/verif.nc, output/data/runs/b30a-0209f019/202405030000/verif.nc, output/data/runs/b30a-0209f019/202408030000/verif.nc, output/data/runs/b30a-0209f019/202410060000/verif.nc, output/data/runs/b30a-0209f019/202411300000/verif.nc, output/data/runs/b30a-0209f019/202412050000/verif.nc, output/data/runs/b30a-0209f019/202411240000/verif.nc, output/data/runs/b30a-0209f019/202403290000/verif.nc, output/data/runs/b30a-0209f019/202408110000/verif.nc, output/data/runs/b30a-0209f019/202410280000/verif.nc, output/data/runs/b30a-0209f019/202404090000/verif.nc, output/data/runs/b30a-0209f019/202411170000/verif.nc, output/data/runs/b30a-0209f019/202406110000/verif.nc, output/data/runs/b30a-0209f019/202406090000/verif.nc, output/data/runs/b30a-0209f019/202410240000/verif.nc, output/data/runs/b30a-0209f019/202401100000/verif.nc, output/data/runs/b30a-0209f019/202410130000/verif.nc, output/data/runs/b30a-0209f019/202402060000/verif.nc, output/data/runs/b30a-0209f019/202401060000/verif.nc, output/data/runs/b30a-0209f019/202406050000/verif.nc, output/data/runs/b30a-0209f019/202404080000/verif.nc, output/data/runs/b30a-0209f019/202403220000/verif.nc, output/data/runs/b30a-0209f019/202401170000/verif.nc, output/data/runs/b30a-0209f019/202403030000/verif.nc, output/data/runs/b30a-0209f019/202408120000/verif.nc, output/data/runs/b30a-0209f019/202411180000/verif.nc, output/data/runs/b30a-0209f019/202402230000/verif.nc, output/data/runs/b30a-0209f019/202406070000/verif.nc, output/data/runs/b30a-0209f019/202409060000/verif.nc, output/data/runs/b30a-0209f019/202408150000/verif.nc, output/data/runs/b30a-0209f019/202412170000/verif.nc, output/data/runs/b30a-0209f019/202403170000/verif.nc, output/data/runs/b30a-0209f019/202402100000/verif.nc, output/data/runs/b30a-0209f019/202411220000/verif.nc, output/data/runs/b30a-0209f019/202409160000/verif.nc, output/data/runs/b30a-0209f019/202411280000/verif.nc, output/data/runs/b30a-0209f019/202409290000/verif.nc, output/data/runs/b30a-0209f019/202402240000/verif.nc, output/data/runs/b30a-0209f019/202408220000/verif.nc, output/data/runs/b30a-0209f019/202408270000/verif.nc, output/data/runs/b30a-0209f019/202405040000/verif.nc, output/data/runs/b30a-0209f019/202412010000/verif.nc, output/data/runs/b30a-0209f019/202402150000/verif.nc, output/data/runs/b30a-0209f019/202404030000/verif.nc, output/data/runs/b30a-0209f019/202405310000/verif.nc, output/data/runs/b30a-0209f019/202410120000/verif.nc, output/data/runs/b30a-0209f019/202402120000/verif.nc, output/data/runs/b30a-0209f019/202404040000/verif.nc, output/data/runs/b30a-0209f019/202411290000/verif.nc, output/data/runs/b30a-0209f019/202411030000/verif.nc, output/data/runs/b30a-0209f019/202406080000/verif.nc, output/data/runs/b30a-0209f019/202402250000/verif.nc, output/data/runs/b30a-0209f019/202405120000/verif.nc, output/data/runs/b30a-0209f019/202410290000/verif.nc, output/data/runs/b30a-0209f019/202410150000/verif.nc, output/data/runs/b30a-0209f019/202408130000/verif.nc, output/data/runs/b30a-0209f019/202408300000/verif.nc, output/data/runs/b30a-0209f019/202404010000/verif.nc, output/data/runs/b30a-0209f019/202404050000/verif.nc, output/data/runs/b30a-0209f019/202411110000/verif.nc, output/data/runs/b30a-0209f019/202408200000/verif.nc, output/data/runs/b30a-0209f019/202408050000/verif.nc, output/data/runs/b30a-0209f019/202405210000/verif.nc, output/data/runs/b30a-0209f019/202409260000/verif.nc, output/data/runs/b30a-0209f019/202405010000/verif.nc, output/data/runs/b30a-0209f019/202402200000/verif.nc, output/data/runs/b30a-0209f019/202403160000/verif.nc, output/data/runs/b30a-0209f019/202401070000/verif.nc, output/data/runs/b30a-0209f019/202409240000/verif.nc, output/data/runs/b30a-0209f019/202409010000/verif.nc, output/data/runs/b30a-0209f019/202402210000/verif.nc, output/data/runs/b30a-0209f019/202409080000/verif.nc, output/data/runs/b30a-0209f019/202406030000/verif.nc, output/data/runs/b30a-0209f019/202409040000/verif.nc, output/data/runs/b30a-0209f019/202408190000/verif.nc, output/data/runs/b30a-0209f019/202404210000/verif.nc, output/data/runs/b30a-0209f019/202402010000/verif.nc, output/data/runs/b30a-0209f019/202407220000/verif.nc, output/data/runs/b30a-0209f019/202404110000/verif.nc, output/data/runs/b30a-0209f019/202406290000/verif.nc, output/data/runs/b30a-0209f019/202411010000/verif.nc, output/data/runs/b30a-0209f019/202404230000/verif.nc, output/data/runs/b30a-0209f019/202411210000/verif.nc, output/data/runs/b30a-0209f019/202407080000/verif.nc, output/data/runs/b30a-0209f019/202412240000/verif.nc, output/data/runs/b30a-0209f019/202406150000/verif.nc, output/data/runs/b30a-0209f019/202409230000/verif.nc, output/data/runs/b30a-0209f019/202403310000/verif.nc, output/data/runs/b30a-0209f019/202406250000/verif.nc, output/data/runs/b30a-0209f019/202407150000/verif.nc, output/data/runs/b30a-0209f019/202410200000/verif.nc, output/data/runs/b30a-0209f019/202406160000/verif.nc, output/data/runs/b30a-0209f019/202402110000/verif.nc, output/data/runs/b30a-0209f019/202401020000/verif.nc, output/data/runs/b30a-0209f019/202403150000/verif.nc, output/data/runs/b30a-0209f019/202407030000/verif.nc, output/data/runs/b30a-0209f019/202402070000/verif.nc, output/data/runs/b30a-0209f019/202408240000/verif.nc, output/data/runs/b30a-0209f019/202407090000/verif.nc, output/data/runs/b30a-0209f019/202403040000/verif.nc, output/data/runs/b30a-0209f019/202406020000/verif.nc, output/data/runs/b30a-0209f019/202409070000/verif.nc, output/data/runs/b30a-0209f019/202410250000/verif.nc, output/data/runs/b30a-0209f019/202405220000/verif.nc, output/data/runs/b30a-0209f019/202409090000/verif.nc, output/data/runs/b30a-0209f019/202402040000/verif.nc, output/data/runs/b30a-0209f019/202405230000/verif.nc, output/data/runs/b30a-0209f019/202408290000/verif.nc, output/data/runs/b30a-0209f019/202407130000/verif.nc, output/data/runs/b30a-0209f019/202404250000/verif.nc, output/data/runs/b30a-0209f019/202406260000/verif.nc, output/data/runs/b30a-0209f019/202410100000/verif.nc, output/data/runs/b30a-0209f019/202411270000/verif.nc, output/data/runs/b30a-0209f019/202404270000/verif.nc, output/data/runs/b30a-0209f019/202405070000/verif.nc, output/data/runs/b30a-0209f019/202401140000/verif.nc, output/data/runs/b30a-0209f019/202412100000/verif.nc, output/data/runs/b30a-0209f019/202409140000/verif.nc, output/data/runs/b30a-0209f019/202403270000/verif.nc, output/data/runs/b30a-0209f019/202411080000/verif.nc, output/data/runs/b30a-0209f019/202409020000/verif.nc, output/data/runs/b30a-0209f019/202409030000/verif.nc, output/data/runs/b30a-0209f019/202411100000/verif.nc, output/data/runs/b30a-0209f019/202408170000/verif.nc, output/data/runs/b30a-0209f019/202403020000/verif.nc, output/data/runs/b30a-0209f019/202408100000/verif.nc, output/data/runs/b30a-0209f019/202403060000/verif.nc, output/data/runs/b30a-0209f019/202407110000/verif.nc, output/data/runs/b30a-0209f019/202403090000/verif.nc, output/data/runs/b30a-0209f019/202402260000/verif.nc, output/data/runs/b30a-0209f019/202402140000/verif.nc, output/data/runs/b30a-0209f019/202404180000/verif.nc, output/data/runs/b30a-0209f019/202401190000/verif.nc, output/data/runs/b30a-0209f019/202408260000/verif.nc, output/data/runs/b30a-0209f019/202405100000/verif.nc, output/data/runs/b30a-0209f019/202408160000/verif.nc, output/data/runs/b30a-0209f019/202402170000/verif.nc, output/data/runs/b30a-0209f019/202404220000/verif.nc, output/data/runs/b30a-0209f019/202405240000/verif.nc, output/data/runs/b30a-0209f019/202407310000/verif.nc, output/data/runs/b30a-0209f019/202404160000/verif.nc, output/data/runs/b30a-0209f019/202408310000/verif.nc, output/data/runs/b30a-0209f019/202404300000/verif.nc, output/data/runs/b30a-0209f019/202407250000/verif.nc, output/data/runs/b30a-0209f019/202408070000/verif.nc, output/data/runs/b30a-0209f019/202406140000/verif.nc, output/data/runs/b30a-0209f019/202407100000/verif.nc, output/data/runs/b30a-0209f019/202403080000/verif.nc, output/data/runs/b30a-0209f019/202407070000/verif.nc, output/data/runs/b30a-0209f019/202407210000/verif.nc, output/data/runs/b30a-0209f019/202403280000/verif.nc, output/data/runs/b30a-0209f019/202402020000/verif.nc, output/data/runs/b30a-0209f019/202410140000/verif.nc, output/data/runs/b30a-0209f019/202411160000/verif.nc, output/data/runs/b30a-0209f019/202405110000/verif.nc, output/data/runs/b30a-0209f019/202410070000/verif.nc, output/data/runs/b30a-0209f019/202404120000/verif.nc, output/data/runs/b30a-0209f019/202401290000/verif.nc, output/data/runs/b30a-0209f019/202411140000/verif.nc, output/data/runs/b30a-0209f019/202401110000/verif.nc, output/data/runs/b30a-0209f019/202412190000/verif.nc, output/data/runs/b30a-0209f019/202408230000/verif.nc, output/data/runs/b30a-0209f019/202403200000/verif.nc, output/data/runs/b30a-0209f019/202401200000/verif.nc, output/data/runs/b30a-0209f019/202405270000/verif.nc, output/data/runs/b30a-0209f019/202405250000/verif.nc, output/data/runs/b30a-0209f019/202402280000/verif.nc, output/data/runs/b30a-0209f019/202409210000/verif.nc, output/data/runs/b30a-0209f019/202412140000/verif.nc, output/data/runs/b30a-0209f019/202405170000/verif.nc, output/data/runs/b30a-0209f019/202402160000/verif.nc, output/data/runs/b30a-0209f019/202401210000/verif.nc, output/data/runs/b30a-0209f019/202406170000/verif.nc, output/data/runs/b30a-0209f019/202406010000/verif.nc, output/data/runs/b30a-0209f019/202405050000/verif.nc, output/data/runs/b30a-0209f019/202404130000/verif.nc, output/data/runs/b30a-0209f019/202401310000/verif.nc, output/data/runs/b30a-0209f019/202412080000/verif.nc, output/data/runs/b30a-0209f019/202403300000/verif.nc, output/data/runs/b30a-0209f019/202401080000/verif.nc, output/data/runs/b30a-0209f019/202412220000/verif.nc, output/data/runs/b30a-0209f019/202411260000/verif.nc, output/data/runs/b30a-0209f019/202409300000/verif.nc, output/data/runs/b30a-0209f019/202405190000/verif.nc, output/data/runs/b30a-0209f019/202405020000/verif.nc, output/data/runs/b30a-0209f019/202408210000/verif.nc, output/data/runs/b30a-0209f019/202401260000/verif.nc, output/data/runs/b30a-0209f019/202405140000/verif.nc, output/data/runs/b30a-0209f019/202401180000/verif.nc, output/data/runs/b30a-0209f019/202407260000/verif.nc, output/data/runs/b30a-0209f019/202411070000/verif.nc, output/data/runs/b30a-0209f019/202409110000/verif.nc, output/data/runs/b30a-0209f019/202403010000/verif.nc, output/data/runs/b30a-0209f019/202404060000/verif.nc, output/data/runs/b30a-0209f019/202404170000/verif.nc, output/data/runs/b30a-0209f019/202407290000/verif.nc, output/data/runs/b30a-0209f019/202409130000/verif.nc, output/data/runs/b30a-0209f019/202403070000/verif.nc, output/data/runs/b30a-0209f019/202404140000/verif.nc, output/data/runs/b30a-0209f019/202402180000/verif.nc, output/data/runs/b30a-0209f019/202410020000/verif.nc, output/data/runs/b30a-0209f019/202409150000/verif.nc, output/data/runs/b30a-0209f019/202407120000/verif.nc, output/data/runs/b30a-0209f019/202411040000/verif.nc, output/data/runs/b30a-0209f019/202409280000/verif.nc, output/data/runs/b30a-0209f019/202412020000/verif.nc, output/data/runs/b30a-0209f019/202406210000/verif.nc, output/data/runs/b30a-0209f019/202403100000/verif.nc, output/data/runs/b30a-0209f019/202406180000/verif.nc, output/data/runs/b30a-0209f019/202404290000/verif.nc, output/data/runs/b30a-0209f019/202401130000/verif.nc, output/data/runs/b30a-0209f019/202408250000/verif.nc, output/data/runs/b30a-0209f019/202402080000/verif.nc, output/data/runs/b30a-0209f019/202411250000/verif.nc, output/data/runs/b30a-0209f019/202409180000/verif.nc, output/data/runs/b30a-0209f019/202410270000/verif.nc, output/data/runs/b30a-0209f019/202411090000/verif.nc, output/data/runs/b30a-0209f019/202402270000/verif.nc, output/data/runs/b30a-0209f019/202410030000/verif.nc, output/data/runs/b30a-0209f019/202406240000/verif.nc, output/data/runs/b30a-0209f019/202408280000/verif.nc, output/data/runs/b30a-0209f019/202409050000/verif.nc, output/data/runs/b30a-0209f019/202404240000/verif.nc, output/data/runs/b30a-0209f019/202402130000/verif.nc, output/data/runs/b30a-0209f019/202407160000/verif.nc, output/data/runs/b30a-0209f019/202410160000/verif.nc, output/data/runs/b30a-0209f019/202401250000/verif.nc, output/data/runs/b30a-0209f019/202405090000/verif.nc, output/data/runs/b30a-0209f019/202410010000/verif.nc, output/data/runs/b30a-0209f019/202410210000/verif.nc, output/data/runs/b30a-0209f019/202403120000/verif.nc, output/data/runs/b30a-0209f019/202401280000/verif.nc, output/data/runs/b30a-0209f019/202404150000/verif.nc, output/data/runs/b30a-0209f019/202403240000/verif.nc, output/data/runs/b30a-0209f019/202403210000/verif.nc, output/data/runs/b30a-0209f019/202411050000/verif.nc, output/data/runs/b30a-0209f019/202406230000/verif.nc, output/data/runs/b30a-0209f019/202403260000/verif.nc, output/data/runs/b30a-0209f019/202409200000/verif.nc, output/data/runs/b30a-0209f019/202410110000/verif.nc, output/data/runs/b30a-0209f019/202404200000/verif.nc, output/data/runs/b30a-0209f019/202401030000/verif.nc, output/data/runs/b30a-0209f019/202405130000/verif.nc, output/data/runs/b30a-0209f019/202407170000/verif.nc, output/data/runs/b30a-0209f019/202406100000/verif.nc, output/data/runs/b30a-0209f019/202410310000/verif.nc, output/data/runs/b30a-0209f019/202401240000/verif.nc, output/data/runs/b30a-0209f019/202402030000/verif.nc, output/data/runs/b30a-0209f019/202409220000/verif.nc, output/data/runs/b30a-0209f019/202403230000/verif.nc, output/data/runs/b30a-0209f019/202410230000/verif.nc, output/data/runs/b30a-0209f019/202405290000/verif.nc, output/data/runs/b30a-0209f019/202408180000/verif.nc, output/data/runs/b30a-0209f019/202402090000/verif.nc, output/data/runs/b30a-0209f019/202405200000/verif.nc, output/data/runs/b30a-0209f019/202403110000/verif.nc, output/data/runs/b30a-0209f019/202403140000/verif.nc, output/data/runs/b30a-0209f019/202406190000/verif.nc, output/data/runs/b30a-0209f019/202412160000/verif.nc, output/data/runs/b30a-0209f019/202412230000/verif.nc, output/data/runs/b30a-0209f019/202406060000/verif.nc, output/data/runs/b30a-0209f019/202410050000/verif.nc, output/data/runs/b30a-0209f019/202409270000/verif.nc, output/data/runs/b30a-0209f019/202401160000/verif.nc, output/data/runs/b30a-0209f019/202411020000/verif.nc, output/data/runs/b30a-0209f019/202410220000/verif.nc, output/data/runs/b30a-0209f019/202412200000/verif.nc, output/data/runs/b30a-0209f019/202410190000/verif.nc, output/data/runs/b30a-0209f019/202404190000/verif.nc, output/data/runs/b30a-0209f019/202410300000/verif.nc, output/data/runs/b30a-0209f019/202405180000/verif.nc, output/data/runs/b30a-0209f019/202403190000/verif.nc, output/data/runs/b30a-0209f019/202412130000/verif.nc, output/data/runs/b30a-0209f019/202403180000/verif.nc, output/data/runs/b30a-0209f019/202411190000/verif.nc, output/data/runs/b30a-0209f019/202405060000/verif.nc, output/data/runs/b30a-0209f019/202402220000/verif.nc, output/data/runs/b30a-0209f019/202410080000/verif.nc, output/data/runs/b30a-0209f019/202412250000/verif.nc, output/data/runs/b30a-0209f019/202410180000/verif.nc, output/data/runs/b30a-0209f019/202404020000/verif.nc, output/data/runs/b30a-0209f019/202404100000/verif.nc, output/data/runs/b30a-0209f019/202408010000/verif.nc, output/data/runs/b30a-0209f019/202401050000/verif.nc, output/data/runs/b30a-0209f019/202404280000/verif.nc, output/data/runs/b30a-0209f019/202406300000/verif.nc, output/data/runs/b30a-0209f019/202411150000/verif.nc, output/data/runs/b30a-0209f019/202407300000/verif.nc, output/data/runs/b30a-0209f019/202412090000/verif.nc, output/data/runs/b30a-0209f019/202409170000/verif.nc, output/data/runs/b30a-0209f019/202405260000/verif.nc, output/data/runs/b30a-0209f019/202411060000/verif.nc, output/data/runs/b30a-0209f019/202408140000/verif.nc, output/data/runs/b30a-0209f019/202412150000/verif.nc, output/data/runs/b30a-0209f019/202408020000/verif.nc, output/data/runs/b30a-0209f019/202412070000/verif.nc, output/data/runs/b30a-0209f019/202401230000/verif.nc, output/data/runs/b30a-0209f019/202411130000/verif.nc, output/data/runs/b30a-0209f019/202407140000/verif.nc, output/data/runs/b30a-0209f019/202409120000/verif.nc, output/data/runs/b30a-0209f019/202409250000/verif.nc, output/data/runs/b30a-0209f019/202401150000/verif.nc, output/data/runs/b30a-0209f019/202412180000/verif.nc, output/data/runs/b30a-0209f019/202412060000/verif.nc, output/data/runs/b30a-0209f019/202412040000/verif.nc, output/data/runs/b30a-0209f019/202411230000/verif.nc, output/data/runs/b30a-0209f019/202404260000/verif.nc, output/data/runs/b30a-0209f019/202407050000/verif.nc, output/data/runs/b30a-0209f019/202403250000/verif.nc, output/data/runs/b30a-0209f019/202402050000/verif.nc, output/data/runs/b30a-0209f019/202407040000/verif.nc, output/data/runs/b30a-0209f019/202406270000/verif.nc, output/data/runs/b30a-0209f019/202402290000/verif.nc, output/data/runs/b30a-0209f019/202406130000/verif.nc, output/data/runs/b30a-0209f019/202406220000/verif.nc, output/data/runs/b30a-0209f019/202407020000/verif.nc, output/data/runs/b30a-0209f019/202403050000/verif.nc, output/data/runs/b30a-0209f019/202408040000/verif.nc, output/data/runs/b30a-0209f019/202406120000/verif.nc, output/data/runs/b30a-0209f019/202411120000/verif.nc, output/data/runs/b30a-0209f019/202407240000/verif.nc, output/data/runs/b30a-0209f019/202412030000/verif.nc, output/data/runs/b30a-0209f019/202410040000/verif.nc, output/data/runs/b30a-0209f019/202401300000/verif.nc, output/data/runs/b30a-0209f019/202412110000/verif.nc, output/data/runs/b30a-0209f019/202407270000/verif.nc, output/data/runs/b30a-0209f019/202405300000/verif.nc, output/data/runs/b30a-0209f019/202407190000/verif.nc, output/data/runs/b30a-0209f019/202406280000/verif.nc, output/data/runs/b30a-0209f019/202410090000/verif.nc, output/data/runs/b30a-0209f019/202401220000/verif.nc, output/data/runs/b30a-0209f019/202401090000/verif.nc, output/data/runs/b30a-0209f019/202409100000/verif.nc, output/data/runs/b30a-0209f019/202408080000/verif.nc, output/data/runs/b30a-0209f019/202406200000/verif.nc, output/data/runs/b30a-0209f019/202407010000/verif.nc, output/data/runs/b30a-0209f019/202403130000/verif.nc, output/data/runs/b30a-0209f019/202401270000/verif.nc, output/data/runs/b30a-0209f019/202412210000/verif.nc, output/data/runs/b30a-0209f019/202405150000/verif.nc, output/data/runs/b30a-0209f019/202409190000/verif.nc, output/data/runs/b30a-0209f019/202407200000/verif.nc, output/data/runs/b30a-0209f019/202405280000/verif.nc, output/data/runs/b30a-0209f019/202401120000/verif.nc, output/data/runs/b30a-0209f019/202404070000/verif.nc, output/data/runs/b30a-0209f019/202402190000/verif.nc, output/data/runs/b30a-0209f019/202405080000/verif.nc, output/data/runs/b30a-0209f019/202412120000/verif.nc, output/data/runs/b30a-0209f019/202405160000/verif.nc, output/data/runs/b30a-0209f019/202411200000/verif.nc + wildcards: run_id=b30a-0209f019 + resources: mem_mb=250000, mem_mib=238419, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=24, mem_mb_per_cpu=1800, runtime=1440, slurm_extra=--exclude=nid001229,nid001225,nid001226,nid001227,nid001230 +Would remove temporary output output/data/runs/b30a-0209f019/202401020000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401030000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401040000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401050000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401060000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401070000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401080000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401090000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401100000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401110000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401120000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401130000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401140000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401150000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401160000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401170000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401180000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401190000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401200000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401210000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401220000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401230000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401240000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401250000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401260000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401270000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401280000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401290000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401300000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202401310000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402010000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402020000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402030000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402040000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402050000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402060000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402070000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402080000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402090000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402100000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402110000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402120000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402130000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402140000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402150000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402160000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402170000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402180000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402190000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402200000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402210000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402220000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402230000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402240000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402250000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402260000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402270000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402280000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202402290000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403010000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403020000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403030000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403040000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403050000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403060000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403070000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403080000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403090000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403100000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403110000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403120000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403130000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403140000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403150000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403160000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403170000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403180000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403190000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403200000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403210000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403220000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403230000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403240000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403250000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403260000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403270000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403280000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403290000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403300000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202403310000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404010000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404020000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404030000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404040000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404050000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404060000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404070000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404080000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404090000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404100000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404110000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404120000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404130000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404140000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404150000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404160000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404170000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404180000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404190000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404200000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404210000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404220000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404230000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404240000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404250000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404260000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404270000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404280000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404290000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202404300000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405010000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405020000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405030000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405040000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405050000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405060000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405070000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405080000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405090000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405100000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405110000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405120000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405130000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405140000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405150000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405160000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405170000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405180000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405190000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405200000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405210000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405220000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405230000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405240000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405250000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405260000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405270000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405280000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405290000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405300000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202405310000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406010000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406020000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406030000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406040000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406050000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406060000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406070000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406080000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406090000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406100000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406110000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406120000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406130000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406140000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406150000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406160000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406170000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406180000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406190000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406200000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406210000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406220000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406230000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406240000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406250000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406260000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406270000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406280000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406290000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202406300000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407010000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407020000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407030000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407040000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407050000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407060000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407070000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407080000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407090000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407100000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407110000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407120000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407130000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407140000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407150000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407160000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407170000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407180000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407190000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407200000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407210000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407220000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407230000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407240000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407250000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407260000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407270000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407280000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407290000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407300000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202407310000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408010000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408020000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408030000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408040000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408050000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408060000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408070000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408080000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408090000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408100000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408110000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408120000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408130000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408140000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408150000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408160000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408170000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408180000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408190000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408200000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408210000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408220000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408230000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408240000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408250000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408260000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408270000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408280000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408290000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408300000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202408310000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409010000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409020000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409030000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409040000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409050000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409060000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409070000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409080000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409090000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409100000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409110000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409120000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409130000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409140000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409150000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409160000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409170000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409180000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409190000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409200000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409210000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409220000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409230000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409240000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409250000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409260000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409270000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409280000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409290000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202409300000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410010000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410020000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410030000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410040000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410050000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410060000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410070000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410080000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410090000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410100000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410110000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410120000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410130000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410140000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410150000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410160000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410170000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410180000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410190000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410200000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410210000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410220000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410230000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410240000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410250000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410260000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410270000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410280000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410290000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410300000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202410310000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411010000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411020000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411030000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411040000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411050000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411060000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411070000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411080000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411090000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411100000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411110000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411120000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411130000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411140000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411150000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411160000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411170000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411180000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411190000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411200000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411210000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411220000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411230000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411240000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411250000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411260000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411270000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411280000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411290000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202411300000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412010000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412020000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412030000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412040000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412050000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412060000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412070000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412080000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412090000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412100000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412110000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412120000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412130000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412140000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412150000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412160000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412170000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412180000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412190000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412200000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412210000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412220000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412230000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412240000/verif.nc +Would remove temporary output output/data/runs/b30a-0209f019/202412250000/verif.nc + +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_JJA_6.png + jobid: 1448 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_JJA_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc + wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=centraleurope, season=JJA, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule verif_metrics_plot: + input: workflow/scripts/verif_plot_metrics.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc, output/data/runs/b30a-0209f019/verif_aggregated.nc + output: output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/plots + log: output/logs/verif_metrics_plot/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd.log + jobid: 1443 + reason: Missing output files: output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/plots; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + wildcards: experiment=experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd + resources: mem_mb=50000, mem_mib=47684, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=16, mem_mb_per_cpu=1800, runtime=20 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_all_6.png + jobid: 1450 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_all_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc + wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=switzerland, season=all, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_all_6.png + jobid: 1445 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_all_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc + wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=centraleurope, season=all, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_SON_6.png + jobid: 1454 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_SON_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc + wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=switzerland, season=SON, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_DJF_6.png + jobid: 1446 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_DJF_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc + wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=centraleurope, season=DJF, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_MAM_6.png + jobid: 1447 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_MAM_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc + wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=centraleurope, season=MAM, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule report_experiment_dashboard: + input: workflow/scripts/report_experiment_dashboard.py, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc, output/data/runs/b30a-0209f019/verif_aggregated.nc, resources/report/dashboard/template.html.jinja2, resources/report/dashboard/script.js, /scratch/mch/lfrey/2025_SEN_eval_ML/software/evalml/2026_01_27_further_develop_maps_of_forecast_metrics/evalml/config/forecasters-ich1_mod_02_1yr.yaml + output: output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/dashboard + log: output/logs/report_experiment_dashboard/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd.log + jobid: 1 + reason: Missing output files: output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/dashboard; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc, output/data/baselines/ICON-CH1-EPS/verif_aggregated.nc + wildcards: experiment=experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_SON_6.png + jobid: 1449 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_SON_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc + wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=centraleurope, season=SON, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_JJA_6.png + jobid: 1453 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_JJA_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc + wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=switzerland, season=JJA, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_DJF_6.png + jobid: 1451 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_DJF_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc + wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=switzerland, season=DJF, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule plot_summary_stat_maps: + input: workflow/scripts/plot_summary_stat_maps.mo.py, output/data/runs/b30a-0209f019/verif_aggregated.nc + output: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_MAM_6.png + jobid: 1452 + reason: Missing output files: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_MAM_6.png; Input files updated by another job: output/data/runs/b30a-0209f019/verif_aggregated.nc + wildcards: experiment=6b9203cd, run_id=b30a-0209f019, param=SP_10M, metric=BIAS, region=switzerland, season=MAM, leadtime=6 + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=postproc, cpus_per_task=1, mem_mb_per_cpu=1800, runtime=10 +[Thu Feb 26 12:18:24 2026] +rule experiment_all: + input: output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/dashboard, output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/plots, output/data/runs/b30a-0209f019/summary.md, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_all_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_DJF_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_MAM_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_JJA_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_SON_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_all_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_DJF_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_MAM_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_JJA_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_SON_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_all_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_DJF_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_MAM_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_JJA_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_SON_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_all_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_DJF_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_MAM_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_JJA_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_SON_6.png + jobid: 0 + reason: Input files updated by another job: output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_JJA_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_DJF_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_MAM_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_SON_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_SON_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_MAM_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_DJF_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_all_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_DJF_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_all_6.png, output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/dashboard, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_JJA_6.png, output/data/runs/b30a-0209f019/summary.md, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_DJF_6.png, output/results/experiment/20260226_forecasters-ich1_mod_02_1yr_6b9203cd/metrics/plots, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_SON_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_JJA_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_MAM_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_switzerland_SON_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_centraleurope_MAM_6.png, output/results/6b9203cd/metrics/spatial/runs/b30a-0209f019/SP_10M_BIAS_centraleurope_all_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_all_6.png, output/results/6b9203cd/metrics/spatial/baselines/ICON-CH1-EPS/SP_10M_BIAS_switzerland_JJA_6.png + resources: mem_mb=, disk_mb=, tmpdir=, slurm_partition=, cpus_per_task=1, mem_mb_per_cpu=1800, runtime= +Job stats: +job count +---------------------------------- ------- +execute_inference 359 +experiment_all 1 +make_squashfs_image 1 +plot_summary_stat_maps 10 +plot_summary_stat_maps_baseline 10 +prepare_inference_forecaster 358 +report_experiment_dashboard 1 +verif_metrics 359 +verif_metrics_aggregation 1 +verif_metrics_aggregation_baseline 1 +verif_metrics_baseline 359 +verif_metrics_plot 1 +write_summary 1 +total 1462 + +Reasons: + (check individual jobs above for details) + code has changed since last execution: + prepare_inference_forecaster + input files updated by another job: + execute_inference, experiment_all, plot_summary_stat_maps, plot_summary_stat_maps_baseline, report_experiment_dashboard, verif_metrics, verif_metrics_aggregation, verif_metrics_aggregation_baseline, verif_metrics_plot + output files have to be generated: + make_squashfs_image, plot_summary_stat_maps, plot_summary_stat_maps_baseline, report_experiment_dashboard, verif_metrics, verif_metrics_baseline, verif_metrics_plot + set of input files has changed since last execution: + write_summary + updated input files: + make_squashfs_image, verif_metrics_baseline +Some jobs were triggered by provenance information, see 'reason' section in the rule displays above. +If you prefer that only modification time is used to determine whether a job shall be executed, use the command line option '--rerun-triggers mtime' (also see --help). +If you are sure that a change for a certain output file (say, ) won't change the result (e.g. because you just changed the formatting of a script or environment definition), you can also wipe its metadata to skip such a trigger via 'snakemake --cleanup-metadata '. +Rules with provenance triggered jobs: prepare_inference_forecaster write_summary + +This was a dry-run (flag -n). The order of jobs does not reflect the order of execution. diff --git a/src/plotting/__init__.py b/src/plotting/__init__.py index ce5e4e63..c905fcc3 100644 --- a/src/plotting/__init__.py +++ b/src/plotting/__init__.py @@ -29,12 +29,16 @@ "extent": [-16.0, 25.0, 30.0, 65.0], "projection": _PROJECTIONS["orthographic"], }, + # The domains which are originally called "centraleurope" and "switzerland" + # are mostly the same. I suggest making domain "switzerland" much smaller, + # so that more spatial detail can be seen, especially in the complex + # topography of the alps. "centraleurope": { "extent": [-2.6, 19.5, 40.2, 52.3], "projection": _PROJECTIONS["orthographic"], }, "switzerland": { - "extent": [0, 17.5, 40.5, 53.0], + "extent": [5.5, 11.0, 45.5, 48.0], "projection": _PROJECTIONS["orthographic"], }, } diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 52ff0502..7ad4878a 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -4,13 +4,13 @@ from matplotlib import pyplot as plt import warnings from .colormap_loader import load_ncl_colormap - +from matplotlib.colors import BoundaryNorm +import numpy as np def _fallback(): warnings.warn("No colormap found for this parameter, using fallback.", UserWarning) return {"cmap": plt.get_cmap("viridis"), "norm": None, "units": ""} - _CMAP_DEFAULTS = { "SP": {"cmap": plt.get_cmap("coolwarm", 11), "vmin": 800 * 100, "vmax": 1100 * 100}, "TD_2M": load_ncl_colormap("t2m_29lev.ct"), @@ -44,6 +44,47 @@ def _fallback(): "units": "mm", "levels": [0, 0.05, 0.1, 0.25, 0.5, 1, 1.5, 2, 3, 4, 5, 6, 7, 100], }, + + # hard-code this for the moment, can still make smarter later on: + # RMSE and MAE first (is all the same). Sequential colour map to reflect the nature of the data (error, all positive). + # Red is suggestive of 'bad' (high error). + # Use a limited number of levels so that absolute values of error can be read from the map. + # always start at 0 so that the saturation of the colour corresponds to the error magnitude. + + # RMSE: + "U_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "V_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "SP_10M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "TD_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, + "T_2M.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, + "PMSL.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, + "PS.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, + "TOT_PREC.RMSE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 1.5, 2, 3, 4]} | {"units": "mm"}, + # would ideally want a 6th colour on the high end of the colour scale, but somehow + # matplotlib does not do that -> ? + + # MAE: + "U_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "V_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "SP_10M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "m/s"}, + "TD_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, + "T_2M.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 0.5, 1, 1.5, 2, 2.5, 3]} | {"units": "°C"}, + "PMSL.MAE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, + "PS.MAE.spatial": {"cmap": plt.get_cmap("Reds", 7), "levels": [0, 50, 100, 150, 200, 250, 300, 350]} | {"units": "Pa"}, + "TOT_PREC.MAE.spatial": {"cmap": plt.get_cmap("Reds", 6), "levels": [0, 1, 1.5, 2, 3, 4]} | {"units": "mm"}, + # the levels for precip are a bit on the bright side, but still worth keeping consistent with RMSE. + + # Bias: + # diverging colour scheme for the Bias to reflect the nature of the data (can be positive or negative, symmetric). + # Red-Blue colour scheme for all variables except precipitation, where a Brown-Green scheme is more suggestive. + "U_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, + "V_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, + "SP_10M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 9), "levels": np.arange(start = -2.25, stop = 2.26, step = 0.5)} | {"units": "m/s"}, + "TD_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, + "T_2M.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -2.75, stop = 2.76, step = 0.5)} | {"units": "°C"}, + "PMSL.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -110, stop = 111, step = 20)} | {"units": "Pa"}, + "PS.BIAS.spatial": {"cmap": plt.get_cmap("RdBu_r", 11), "levels": np.arange(start = -110, stop = 111, step = 20)} | {"units": "Pa"}, + "TOT_PREC.BIAS.spatial": {"cmap": plt.get_cmap("BrBG", 9), "levels": [-1, -0.5, -0.25, -0.1, 0.1, 0.25, 0.5, 1]} | {"units": "mm"} } CMAP_DEFAULTS = defaultdict(_fallback, _CMAP_DEFAULTS) diff --git a/src/plotting/compat.py b/src/plotting/compat.py index 665287e0..93634aa3 100644 --- a/src/plotting/compat.py +++ b/src/plotting/compat.py @@ -5,6 +5,7 @@ import geopandas as gpd import numpy as np import pandas as pd +import xarray as xr from meteodatalab import data_source from meteodatalab import grib_decoder from shapely.geometry import MultiPoint @@ -90,3 +91,4 @@ def load_state_from_raw( if key.startswith("field_"): state["fields"][key.removeprefix("field_")] = value return state + diff --git a/src/verification/__init__.py b/src/verification/__init__.py index db70f4fc..4a621652 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -78,24 +78,34 @@ def _compute_scores( prefix="", suffix="", source="", + dim=["x", "y"], ) -> xr.Dataset: """ Compute basic verification metrics between two xarray DataArrays (fcst and obs). Returns a xarray Dataset with the computed metrics. """ - dim = ["x", "y"] if "x" in fcst.dims and "y" in fcst.dims else ["values"] + error = fcst - obs - scores = xr.Dataset( - { - f"{prefix}BIAS{suffix}": error.mean(dim=dim, skipna=True), - f"{prefix}MSE{suffix}": (error**2).mean(dim=dim, skipna=True), - f"{prefix}MAE{suffix}": abs(error).mean(dim=dim, skipna=True), - f"{prefix}VAR{suffix}": error.var(dim=dim, skipna=True), - f"{prefix}CORR{suffix}": xr.corr(fcst, obs, dim=dim), - f"{prefix}R2{suffix}": xr.corr(fcst, obs, dim=dim) ** 2, - } - ) - scores = scores.expand_dims({"source": [source]}) + if dim == []: + scores = xr.Dataset( + { + f"{prefix}BIAS{suffix}": error, + f"{prefix}MSE{suffix}": (error**2), + f"{prefix}MAE{suffix}": abs(error), + } + ) + else: + scores = xr.Dataset( + { + f"{prefix}BIAS{suffix}": error.mean(dim=dim, skipna=True), + f"{prefix}MSE{suffix}": (error**2).mean(dim=dim, skipna=True), + f"{prefix}MAE{suffix}": abs(error).mean(dim=dim, skipna=True), + f"{prefix}VAR{suffix}": error.var(dim=dim, skipna=True), + f"{prefix}CORR{suffix}": xr.corr(fcst, obs, dim=dim), + f"{prefix}R2{suffix}": xr.corr(fcst, obs, dim=dim) ** 2, + } + ) + # scores = scores.expand_dims({"source": [source]}) return scores @@ -104,12 +114,13 @@ def _compute_statistics( prefix="", suffix="", source="", + dim=["x", "y"], ) -> xr.Dataset: """ Compute basic statistics of a xarray DataArray (data). Returns a xarray Dataset with the computed statistics. """ - dim = ["x", "y"] if "x" in data.dims and "y" in data.dims else ["values"] + stats = xr.Dataset( { f"{prefix}mean{suffix}": data.mean(dim=dim, skipna=True), @@ -153,6 +164,8 @@ def verify( """ start = time.time() + dim = ["x", "y"] if "x" in fcst.dims and "y" in fcst.dims else ["values"] + # rewrite the verification to use dask and xarray # chunk the data to avoid memory issues # compute the metrics in parallel @@ -180,27 +193,36 @@ def verify( # scores vs time (reduce spatially) score.append( _compute_scores( - fcst_param, obs_param, prefix=param + ".", source=fcst_label + fcst_param, obs_param, prefix=param + ".", source=fcst_label, dim=dim ).expand_dims(region=[region]) ) # statistics vs time (reduce spatially) fcst_statistics.append( _compute_statistics( - fcst_param, prefix=param + ".", source=fcst_label + fcst_param, prefix=param + ".", source=fcst_label, dim=dim ).expand_dims(region=[region]) ) obs_statistics.append( _compute_statistics( - obs_param, prefix=param + ".", source=obs_label + obs_param, prefix=param + ".", source=obs_label, dim=dim ).expand_dims(region=[region]) ) score = xr.concat(score, dim="region") fcst_statistics = xr.concat(fcst_statistics, dim="region") obs_statistics = xr.concat(obs_statistics, dim="region") + score_spatial = _compute_scores( + fcst_aligned[param], + obs_aligned[param], + prefix=param + ".", + suffix=".spatial", + dim=[], + ) statistics.append(xr.concat([fcst_statistics, obs_statistics], dim="source")) - scores.append(score) + scores.append( + xr.merge([score, score_spatial], join="outer", compat="no_conflicts") + ) scores = _merge_metrics(scores) statistics = _merge_metrics(statistics) diff --git a/workflow/Snakefile b/workflow/Snakefile index b32f1d62..2c11ecb9 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -126,6 +126,26 @@ rule experiment_all: rules.verif_metrics_plot.output, experiment=EXPERIMENT_NAME, ), + expand( + OUT_ROOT / "results/{experiment}/metrics/spatial/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + run_id=collect_all_candidates(), + leadtime=[6], + metric=["BIAS"], + param=["SP_10M"], + region=["centraleurope", "switzerland"], + season=["all", "DJF", "MAM", "JJA", "SON"], + experiment=EXPERIMENT_NAME + ), + expand( + OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + baseline_id=collect_all_baselines(), + leadtime=[6], + metric=["BIAS"], + param=["SP_10M"], + region=["centraleurope", "switzerland"], + season=["all", "DJF", "MAM", "JJA", "SON"], + experiment=EXPERIMENT_NAME + ) rule showcase_all: diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index 8a3515c7..8ef6e82f 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -115,3 +115,50 @@ rule make_forecast_animation: """ convert -delay {params.delay} -loop 0 {input} {output} """ + + +rule plot_summary_stat_maps: + localrule: True + input: + script="workflow/scripts/plot_summary_stat_maps.mo.py", + verif_file=OUT_ROOT / "data/runs/{run_id}/verif_aggregated.nc", + output: + OUT_ROOT / "results/{experiment}/metrics/spatial/runs/{run_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + wildcard_constraints: + leadtime=r"\d+", # only digits + resources: + slurm_partition="postproc", + cpus_per_task=1, + runtime="10m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" + params: + nc_out_dir=lambda wc: ( + Path(OUT_ROOT) / f"data/runs/{wc.run_id}/verif_aggregated.nc" + # not sure how to do this, because the baselines are in, e.g., output/data/baselines/COSMO-E/verif_aggregated.nc + # and the runs are in output/data/runs/runID/verif_aggregated.nc + ).resolve(), + shell: + """ + export ECCODES_DEFINITION_PATH=$(realpath .venv/share/eccodes-cosmo-resources/definitions) + python {input.script} \ + --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ + --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ + --season {wildcards.season} \ + # interactive editing (needs to set localrule: True and use only one core) + # marimo edit {input.script} -- \ + # --input {input.verif_file} --outfn {output[0]} --region {wildcards.region} \ + # --param {wildcards.param} --leadtime {wildcards.leadtime} --metric {wildcards.metric} \ + # --season {wildcards.season} \ + """ + +use rule plot_summary_stat_maps as plot_summary_stat_maps_baseline with: + input: + script="workflow/scripts/plot_summary_stat_maps.mo.py", + verif_file=OUT_ROOT / "data/baselines/{baseline_id}/verif_aggregated.nc", + output: + OUT_ROOT / "results/{experiment}/metrics/spatial/baselines/{baseline_id}/{param}_{metric}_{region}_{season}_{leadtime}.png", + params: + nc_out_dir=lambda wc: ( + Path(OUT_ROOT) / f"data/baselines/{wc.baseline_id}/verif_aggregated.nc" + # not sure if this is actually needed. Verification file is already specified above as input. Leave it for the time being. + ).resolve() \ No newline at end of file diff --git a/workflow/rules/verif.smk b/workflow/rules/verif.smk index f30ad9a2..36c9a9b0 100644 --- a/workflow/rules/verif.smk +++ b/workflow/rules/verif.smk @@ -34,6 +34,7 @@ rule verif_metrics_baseline: cpus_per_task=24, mem_mb=50_000, runtime="60m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} \ @@ -81,6 +82,7 @@ rule verif_metrics: cpus_per_task=24, mem_mb=50_000, runtime="60m", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} \ @@ -117,7 +119,8 @@ rule verif_metrics_aggregation: resources: cpus_per_task=24, mem_mb=250_000, - runtime="2h", + runtime="24h", + slurm_extra="--exclude=nid001229,nid001225,nid001226,nid001227,nid001230" shell: """ uv run {input.script} {input.verif_nc} --output {output} > {log} 2>&1 diff --git a/workflow/scripts/plot_summary_stat_maps.mo.py b/workflow/scripts/plot_summary_stat_maps.mo.py new file mode 100644 index 00000000..728f49db --- /dev/null +++ b/workflow/scripts/plot_summary_stat_maps.mo.py @@ -0,0 +1,267 @@ +import marimo + +__generated_with = "0.19.4" +app = marimo.App(width="medium") + + +@app.cell +def _(): + + # this sure stays the same. + import logging + from argparse import ArgumentParser + from pathlib import Path + + # this sure stays the same. + import cartopy.crs as ccrs + import earthkit.plots as ekp + import numpy as np + import xarray as xr + + # this stays the same as well. + from plotting import DOMAINS + + # no changes to StatePlotter required according to ChatGPT. + from plotting import StatePlotter + + # Added some new colour maps for the Bias / MAE / RMSE map plots. + from plotting.colormap_defaults import CMAP_DEFAULTS + return ( + ArgumentParser, + CMAP_DEFAULTS, + DOMAINS, + Path, + StatePlotter, + ekp, + logging, + np, + xr, + ) + + +@app.cell +def _(logging): + LOG = logging.getLogger(__name__) + LOG_FMT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + logging.basicConfig(level=logging.INFO, format=LOG_FMT) + return (LOG,) + + +@app.cell +def _(ArgumentParser, Path, np): + parser = ArgumentParser() + + parser.add_argument( + "--input", type=str, default=None, help="Directory to .nc data containing the error fields" + ) + parser.add_argument("--outfn", type=str, help="output filename") + parser.add_argument("--leadtime", type=str, help="leadtime") + parser.add_argument("--param", type=str, help="parameter") + parser.add_argument("--region", type=str, help="name of region") + parser.add_argument("--metric", type=str, help = "Evaluation Metric. So far Bias, RMSE or MAE are implemented.") + parser.add_argument("--season", type=str, default="all", help="season filter") + parser.add_argument("--init_hour", type=str, default="all", help="initialization hour filter") + + args = parser.parse_args() + verif_file = Path(args.input) + outfn = Path(args.outfn) + lead_time = args.leadtime + param = args.param + region = args.region + season = args.season + init_hour = args.init_hour + metric = args.metric + + if isinstance(init_hour, str): + if init_hour == "all": + init_hour = -999 + else: + raise ValueError("init_hour must be 'all' or an integer hour") + + lead_time = np.timedelta64(lead_time, 'h') + return ( + init_hour, + lead_time, + metric, + outfn, + param, + region, + season, + verif_file, + ) + + +@app.cell +def _(init_hour, lead_time, metric, param, season, verif_file, xr): + ds = xr.open_dataset(verif_file) + var = f"{param}.{metric}.spatial" + ds = ds[var].sel(init_hour=init_hour, lead_time=lead_time, season=season) + ds + return ds, var + +@app.cell +def _(CMAP_DEFAULTS, ekp): + def get_style(param, metric, units_override=None): + """Get style and colormap settings for the plot. + Needed because cmap/norm does not work in Style(colors=cmap), + still needs to be passed as arguments to tripcolor()/tricontourf(). + """ + metric_key = f"{param}.{metric}.spatial" + cfg = CMAP_DEFAULTS[metric_key] if metric_key in CMAP_DEFAULTS else CMAP_DEFAULTS[param] + units = units_override if units_override is not None else cfg.get("units", "") + return { + "style": ekp.styles.Style( + levels=cfg.get("bounds", cfg.get("levels", None)), + extend="both", + units=units, + colors=cfg.get("colors", None), + ), + "norm": cfg.get("norm", None), + "cmap": cfg.get("cmap", None), + "levels": cfg.get("levels", None), + "vmin": cfg.get("vmin", None), + "vmax": cfg.get("vmax", None), + "colors": cfg.get("colors", None), + } + return (get_style,) + +# @app.cell +# def _(LOG, np): +# """Preprocess fields with pint-based unit conversion and derived quantities.""" +# try: +# import pint # type: ignore + +# _ureg = pint.UnitRegistry() + +# def _k_to_c(arr): +# # robust conversion with pint, fallback if dtype unsupported +# try: +# return (_ureg.Quantity(arr, _ureg.kelvin).to(_ureg.degC)).magnitude +# except Exception: +# return arr - 273.15 + +# def _ms_to_knots(arr): +# # robust conversion with pint, fallback if dtype unsupported +# try: +# return ( +# _ureg.Quantity(arr, _ureg.meter / _ureg.second).to(_ureg.knot) +# ).magnitude +# except Exception: +# return arr * 1.943844 + +# def _m_to_mm(arr): +# # robust conversion with pint, fallback if dtype unsupported +# try: +# return (_ureg.Quantity(arr, _ureg.meter).to(_ureg.millimeter)).magnitude +# except Exception: +# return arr * 1000 + +# except Exception: +# LOG.warning("pint not available; falling back hardcoded conversions") + +# def _k_to_c(arr): +# return arr - 273.15 + +# def _ms_to_knots(arr): +# return arr * 1.943844 + +# def _m_to_mm(arr): +# return arr * 1000 + +# def preprocess_field(param: str, state: dict): +# """ +# - Temperatures: K -> °C +# - Wind speed: sqrt(u^2 + v^2) +# - Precipitation: m -> mm +# Returns: (field_array, units_override or None) +# """ +# fields = state["fields"] +# # temperature variables +# if param in ("T_2M", "TD_2M", "T", "TD"): +# return _k_to_c(fields[param]), "°C" +# # 10m wind speed (allow legacy 'uv' alias) +# if param == "SP_10M": +# u = fields["U_10M"] +# v = fields["V_10M"] +# return np.sqrt(u**2 + v**2), "m/s" +# # wind speed from standard-level components +# if param == "SP": +# u = fields["U"] +# v = fields["V"] +# return np.sqrt(u**2 + v**2), "m/s" +# if param == "TOT_PREC": +# return _m_to_mm(fields[param]), "mm" +# # default: passthrough +# return fields[param], None + +# return (preprocess_field,) + +@app.cell +def _( + DOMAINS, + LOG, + StatePlotter, + ds, + get_style, + lead_time, + metric, + outfn, + param, + region, + season, + var, +): + # plot individual fields + import matplotlib.pyplot as plt + + plotter = StatePlotter( + ds["longitude"].values.ravel(), + ds["latitude"].values.ravel(), + outfn.parent, + ) + fig = plotter.init_geoaxes( + nrows=1, + ncols=1, + projection=DOMAINS[region]["projection"], + bbox=DOMAINS[region]["extent"], + name=region, + size=(6, 6), + ) + subplot = fig.add_map(row=0, column=0) + + # # preprocess field (unit conversion, derived quantities) + # field, units_override = preprocess_field(param, state) + + # Quick fix for precipitation (might have to use preprocess_field in the end (see above)) + # for wind speed, preprocess_field only has conversion from m/s to knots + # (not the other way around), so I assume the values are in m/s + if param == "TOT_PREC": + plot_vals = ds.values.ravel()*1000 + else: + plot_vals = ds.values.ravel() + + plotter.plot_field(subplot, plot_vals, **get_style(var, metric)) + # subplot.ax.add_geometries( + # state["lam_envelope"], + # edgecolor="black", + # facecolor="none", + # crs=ccrs.PlateCarree(), + # ) + + # validtime = state["valid_time"].strftime("%Y%m%d%H%M") + # # leadtime = int(state["lead_time"].total_seconds() // 3600) + + fig.title(f"{metric} of {param}, Season: {season}, Lead Time: {lead_time}") + + fig.save(outfn, bbox_inches="tight", dpi=200) + LOG.info(f"saved: {outfn}") + return + + +@app.cell +def _(): + return + + +if __name__ == "__main__": + app.run() diff --git a/workflow/scripts/report_experiment_dashboard.py b/workflow/scripts/report_experiment_dashboard.py index aea38fc2..180035ec 100644 --- a/workflow/scripts/report_experiment_dashboard.py +++ b/workflow/scripts/report_experiment_dashboard.py @@ -28,19 +28,81 @@ def program_summary_log(args): LOG.info("=" * 80) +def dataset_metrics_to_dataframe( + ds: xr.Dataset, + forbidden_dims=("values",), + metric_dims=("source", "season", "init_hour", "region", "lead_time", "eps"), +): + """ + Convert a verification xarray.Dataset to a pandas.DataFrame compatible with + the old `to_array("stack").to_dataframe()` workflow. + + Returns columns: + - metric_dims... + - stack (e.g. "T_2M.MAE") + - value + """ + + # keep only non-spatial metric variables + metric_vars = [] + for v in ds.data_vars: + da = ds[v] + + # skip spatial metrics + if "spatial" in v: + continue + + # skip forbidden dimensions + if any(d in da.dims for d in forbidden_dims): + continue + + # only keep vars whose dims are a subset of expected metric dims + if not set(da.dims).issubset(metric_dims): + continue + + metric_vars.append(v) + + if not metric_vars: + raise ValueError("No compatible metric variables found in dataset") + + # stack variables exactly like the original code + df = ( + ds[metric_vars] + .to_array("stack") + .to_dataframe(name="value") + .reset_index() + ) + + return df + def main(args): program_summary_log(args) # Load, de-duplicate lead_time, and keep best provider per source (same logic as verif_plot_metrics) - dfs = [xr.open_dataset(f) for f in args.verif_files] + drop_variables = ["TOT_PREC.MAE.spatial", "TOT_PREC.RMSE.spatial", "TOT_PREC.BIAS.spatial", + "PMSL.MAE.spatial", "PMSL.RMSE.spatial", "PMSL.BIAS.spatial", + "PS.MAE.spatial", "PS.RMSE.spatial", "PS.BIAS.spatial", + "V_10M.MAE.spatial", "V_10M.RMSE.spatial", "V_10M.BIAS.spatial", + "U_10M.MAE.spatial", "U_10M.RMSE.spatial", "U_10M.BIAS.spatial", + "TD_2M.MAE.spatial", "TD_2M.RMSE.spatial", "TD_2M.BIAS.spatial", + "T_2M.MAE.spatial", "T_2M.RMSE.spatial", "T_2M.BIAS.spatial"] + + dfs = [xr.open_dataset(f, drop_variables = drop_variables) for f in args.verif_files] dfs = [_ensure_unique_lead_time(d) for d in dfs] dfs = _select_best_sources(dfs) ds = xr.concat(dfs, dim="source", join="outer") LOG.info("Loaded verification netcdf: \n%s", ds) # extract only non-spatial variables to pd.DataFrame - nonspatial_vars = [d for d in ds.data_vars if "spatial" not in d] - df = ds[nonspatial_vars].to_array("stack").to_dataframe(name="value").reset_index() + # nonspatial_vars = [d for d in ds.data_vars if "spatial" not in d] + # df = ds[nonspatial_vars].to_array("stack").to_dataframe(name="value").reset_index() + + df = dataset_metrics_to_dataframe( + ds, + forbidden_dims=("values",), # critical! + metric_dims=("source", "season", "init_hour", "region", "lead_time", "eps"), + ) + df[["param", "metric"]] = df["stack"].str.split(".", n=1, expand=True) df.drop(columns=["stack"], inplace=True) df["lead_time"] = df["lead_time"].dt.total_seconds() / 3600 diff --git a/workflow/scripts/verif_single_init.py b/workflow/scripts/verif_single_init.py index 5ad505ef..8b1de092 100644 --- a/workflow/scripts/verif_single_init.py +++ b/workflow/scripts/verif_single_init.py @@ -102,9 +102,17 @@ def main(args: ScriptConfig): analysis, ) - # compute metrics and statistics + # before verifying, calculate wind speed: + for ds in [fcst, analysis]: + if "U_10M" in ds and "V_10M" in ds: + LOG.info("Calculating Wind Speed (SP_10M)...") + ds["SP_10M"] = (ds["U_10M"]**2 + ds["V_10M"]**2)**0.5 + # Optional: Add metadata for the netCDF output + ds["SP_10M"].attrs = {"units": "m/s", "long_name": "10m Wind Speed"} + # compute metrics and statistics results = verify(fcst, analysis, args.label, args.analysis_label, args.regions) + LOG.info("Verification results:\n%s", results) # save results to NetCDF args.output.parent.mkdir(parents=True, exist_ok=True)