From f8f3fc8c4c6330a415c9f6a225d5508a8038444c Mon Sep 17 00:00:00 2001 From: Pawan Parida <135092407+zer-art@users.noreply.github.com> Date: Fri, 9 Jan 2026 21:11:24 +0530 Subject: [PATCH 1/2] refactor: Rename configs to uk_gfs_configs and remove non-GFS yamls --- docs/getting_started.md | 10 ++-- run.py | 2 +- .../configs/dwd_data_config.yaml | 42 ---------------- .../met_office_global_data_config.yaml | 50 ------------------- .../configs/met_office_uk_data_config.yaml | 50 ------------------- src/open_data_pvnet/nwp/dwd.py | 2 +- src/open_data_pvnet/nwp/gfs_dataset.py | 2 +- src/open_data_pvnet/nwp/met_office.py | 4 +- src/open_data_pvnet/scripts/save_samples.py | 2 +- .../PVNet_configs/callbacks/default.yaml | 0 .../PVNet_configs/config.yaml | 0 .../configuration/example_configuration.yaml | 0 .../datamodule/premade_batches.yaml | 0 .../datamodule/streamed_batches.yaml | 0 .../experiment/example_simple.yaml | 0 .../PVNet_configs/hydra/default.yaml | 0 .../PVNet_configs/logger/wandb.yaml | 0 .../PVNet_configs/model/multimodal.yaml | 0 .../PVNet_configs/readme.md | 0 .../PVNet_configs/trainer/all_params.yaml | 0 .../PVNet_configs/trainer/default.yaml | 0 .../{configs => uk_gfs_configs}/__init.py | 0 .../gfs_data_config.yaml | 0 23 files changed, 11 insertions(+), 153 deletions(-) delete mode 100644 src/open_data_pvnet/configs/dwd_data_config.yaml delete mode 100644 src/open_data_pvnet/configs/met_office_global_data_config.yaml delete mode 100644 src/open_data_pvnet/configs/met_office_uk_data_config.yaml rename src/open_data_pvnet/{configs => uk_gfs_configs}/PVNet_configs/callbacks/default.yaml (100%) rename src/open_data_pvnet/{configs => uk_gfs_configs}/PVNet_configs/config.yaml (100%) rename src/open_data_pvnet/{configs => uk_gfs_configs}/PVNet_configs/datamodule/configuration/example_configuration.yaml (100%) rename src/open_data_pvnet/{configs => uk_gfs_configs}/PVNet_configs/datamodule/premade_batches.yaml (100%) rename src/open_data_pvnet/{configs => uk_gfs_configs}/PVNet_configs/datamodule/streamed_batches.yaml (100%) rename src/open_data_pvnet/{configs => uk_gfs_configs}/PVNet_configs/experiment/example_simple.yaml (100%) rename src/open_data_pvnet/{configs => uk_gfs_configs}/PVNet_configs/hydra/default.yaml (100%) rename src/open_data_pvnet/{configs => uk_gfs_configs}/PVNet_configs/logger/wandb.yaml (100%) rename src/open_data_pvnet/{configs => uk_gfs_configs}/PVNet_configs/model/multimodal.yaml (100%) rename src/open_data_pvnet/{configs => uk_gfs_configs}/PVNet_configs/readme.md (100%) rename src/open_data_pvnet/{configs => uk_gfs_configs}/PVNet_configs/trainer/all_params.yaml (100%) rename src/open_data_pvnet/{configs => uk_gfs_configs}/PVNet_configs/trainer/default.yaml (100%) rename src/open_data_pvnet/{configs => uk_gfs_configs}/__init.py (100%) rename src/open_data_pvnet/{configs => uk_gfs_configs}/gfs_data_config.yaml (100%) diff --git a/docs/getting_started.md b/docs/getting_started.md index 6bd7416..6bc72eb 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -727,20 +727,20 @@ Common error messages and their solutions: ## Running PVNet Model 1. Update configuration file - Go to src/open_data_pvnet/configs/PVNet_configs/datamodule/streamed_batches.yaml + Go to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/datamodule/streamed_batches.yaml Change values if desired (increase at your discretion): num_train_samples: 5 num_val_samples: 5 -2. Update src/open_data_pvnet/configs/PVNet_configs/datamodule/premade_batches.yaml - Change this line to configuration: +2. Update src/open_data_pvnet/uk_gfs_configs/PVNet_configs/datamodule/premade_batches.yaml + Change this line to configuration: -3. Update src/open_data_pvnet/configs/PVNet_configs/config.yaml +3. Update src/open_data_pvnet/uk_gfs_configs/PVNet_configs/config.yaml Change the line to - datamodule: premade_batches.yaml 4. Open a Weights & Biases Account https://wandb.ai/ - Go to src/open_data_pvnet/configs/PVNet_configs/logger/wandb.yaml + Go to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/logger/wandb.yaml Change to project: "GFS_TEST_RUN" Change to save_dir: "GFS_TEST_RUN" diff --git a/run.py b/run.py index 1dabdbd..14f742e 100644 --- a/run.py +++ b/run.py @@ -39,7 +39,7 @@ @hydra.main( - config_path="src/open_data_pvnet/configs/PVNet_configs", + config_path="src/open_data_pvnet/uk_gfs_configs/PVNet_configs", config_name="config.yaml", version_base="1.2", ) diff --git a/src/open_data_pvnet/configs/dwd_data_config.yaml b/src/open_data_pvnet/configs/dwd_data_config.yaml deleted file mode 100644 index e86b541..0000000 --- a/src/open_data_pvnet/configs/dwd_data_config.yaml +++ /dev/null @@ -1,42 +0,0 @@ -general: - description: Config for accessing DWD (Deutscher Wetterdienst) ICON-EU NWP data - name: PVNet current (DWD) - destination_platform: "huggingface" - destination_dataset_id: "openclimatefix/dwd-icon-eu-deterministic-solar" # Updated dataset ID - -input_data: - default_forecast_minutes: 480 - default_history_minutes: 120 - nwp: - dwd: - dropout_fraction: 1.0 - dropout_timedeltas_minutes: - - -360 - forecast_minutes: 480.0 - history_minutes: 120 - max_staleness_minutes: null - s3_bucket: "opendata.dwd.de" - s3_prefix: "weather/nwp/icon-eu/grib" - local_output_dir: "tmp/dwd" - nwp_accum_channels: - - ASWDIFD_S # Diffuse downward shortwave radiation at surface - - ASWDIR_S # Direct shortwave radiation at surface - - ATHB_S # Net budget of radiation at surface - nwp_channels: - - T_2M # 2m temperature - - ASWDIFD_S # Diffuse downward shortwave radiation at surface - - ASWDIR_S # Direct shortwave radiation at surface - - ATHB_S # Net budget of radiation at surface - - CLCH # High cloud cover - - CLCM # Medium cloud cover - - CLCL # Low cloud cover - - CLCT # Total cloud cover - - H_SNOW # Snow depth - - Z0 # Surface roughness length - - VMAX_10M # Maximum wind speed at 10m - - DD_10M # Wind direction at 10m - nwp_image_size_pixels_height: 1097 # ICON-EU grid dimensions - nwp_image_size_pixels_width: 657 - nwp_provider: dwd - nwp_zarr_path: PLACEHOLDER.zarr - time_resolution_minutes: 60 diff --git a/src/open_data_pvnet/configs/met_office_global_data_config.yaml b/src/open_data_pvnet/configs/met_office_global_data_config.yaml deleted file mode 100644 index 1e8f3eb..0000000 --- a/src/open_data_pvnet/configs/met_office_global_data_config.yaml +++ /dev/null @@ -1,50 +0,0 @@ -general: - description: Config for training the saved PVNet model using Met Office data - name: PVNet current (Met Office) - destination_platform: "huggingface" - destination_dataset_id: null # TODO: Add destination dataset id - -input_data: - default_forecast_minutes: 480 - default_history_minutes: 120 - gsp: - dropout_fraction: 0 - dropout_timedeltas_minutes: null - forecast_minutes: 480 - gsp_zarr_path: PLACEHOLDER.zarr - history_minutes: 120 - time_resolution_minutes: 30 - nwp: - met_office: - dropout_fraction: 1.0 - dropout_timedeltas_minutes: - - -360 - forecast_minutes: 480.0 - history_minutes: 120 - max_staleness_minutes: null - s3_bucket: "met-office-atmospheric-model-data" - s3_prefix: "global-deterministic-10km" - local_output_dir: "tmp/met_office/global" - nwp_accum_channels: - - radiation_flux_in_shortwave_total_downward_at_surface # Downward shortwave radiation flux at the surface (W/m²) - - radiation_flux_in_longwave_downward_at_surface # Downward longwave radiation flux at the surface (W/m²) - - surface_roughness # Surface roughness (m) (if available, otherwise null or computed) - - radiation_flux_in_uv_downward_at_surface # Downward UV radiation flux at the surface (W/m²) - nwp_channels: - - temperature_at_screen_level # 2-meter temperature (°C) - - radiation_flux_in_shortwave_total_downward_at_surface # Downward shortwave radiation flux at the surface (W/m²) - - radiation_flux_in_longwave_downward_at_surface # Downward longwave radiation flux at the surface (W/m²) - - cloud_amount_of_high_cloud # High cloud cover (fraction or %) - - cloud_amount_of_medium_cloud # Medium cloud cover (fraction or %) - - cloud_amount_of_low_cloud # Low cloud cover (fraction or %) - - cloud_amount_of_total_cloud # Total cloud cover (fraction or %) - - snow_depth_water_equivalent # Snow depth equivalent water content (mm) - - surface_roughness # Surface roughness (m) (if available, otherwise null or computed) - - radiation_flux_in_uv_downward_at_surface # Downward UV radiation flux at the surface (W/m²) - - wind_speed_at_10m # 10-meter wind speed (m/s) - - wind_direction_at_10m # 10-meter wind direction (degrees from north) - nwp_image_size_pixels_height: 12 - nwp_image_size_pixels_width: 12 - nwp_provider: met_office - nwp_zarr_path: PLACEHOLDER.zarr - time_resolution_minutes: 60 diff --git a/src/open_data_pvnet/configs/met_office_uk_data_config.yaml b/src/open_data_pvnet/configs/met_office_uk_data_config.yaml deleted file mode 100644 index 1e61e75..0000000 --- a/src/open_data_pvnet/configs/met_office_uk_data_config.yaml +++ /dev/null @@ -1,50 +0,0 @@ -general: - description: Config for accessing Met Office UK Deterministic NWP data - name: PVNet current (Met Office UK) - destination_platform: "huggingface" - destination_dataset_id: "openclimatefix/met-office-uk-deterministic-solar" - -input_data: - default_forecast_minutes: 480 - default_history_minutes: 120 - gsp: - dropout_fraction: 0 - dropout_timedeltas_minutes: null - forecast_minutes: 480 - gsp_zarr_path: PLACEHOLDER.zarr - history_minutes: 120 - time_resolution_minutes: 30 - nwp: - met_office: - dropout_fraction: 1.0 - dropout_timedeltas_minutes: - - -360 - forecast_minutes: 480.0 - history_minutes: 120 - max_staleness_minutes: null - s3_bucket: "met-office-atmospheric-model-data" - s3_prefix: "uk-deterministic-2km" - local_output_dir: "tmp/met_office/uk" - nwp_accum_channels: - - radiation_flux_in_shortwave_total_downward_at_surface # Downward shortwave radiation flux at the surface (W/m²) - - radiation_flux_in_longwave_downward_at_surface # Downward longwave radiation flux at the surface (W/m²) - - surface_roughness # Surface roughness (m) (if available, otherwise null or computed) - - radiation_flux_in_uv_downward_at_surface # Downward UV radiation flux at the surface (W/m²) - nwp_channels: - - temperature_at_screen_level # 2-meter temperature (°C) - - radiation_flux_in_shortwave_total_downward_at_surface # Downward shortwave radiation flux at the surface (W/m²) - - radiation_flux_in_longwave_downward_at_surface # Downward longwave radiation flux at the surface (W/m²) - - cloud_amount_of_high_cloud # High cloud cover (fraction or %) - - cloud_amount_of_medium_cloud # Medium cloud cover (fraction or %) - - cloud_amount_of_low_cloud # Low cloud cover (fraction or %) - - cloud_amount_of_total_cloud # Total cloud cover (fraction or %) - - snow_depth_water_equivalent # Snow depth equivalent water content (mm) - - surface_roughness # Surface roughness (m) (if available, otherwise null or computed) - - radiation_flux_in_uv_downward_at_surface # Downward UV radiation flux at the surface (W/m²) - - wind_speed_at_10m # 10-meter wind speed (m/s) - - wind_direction_at_10m # 10-meter wind direction (degrees from north) - nwp_image_size_pixels_height: 970 - nwp_image_size_pixels_width: 1042 - nwp_provider: met_office - nwp_zarr_path: PLACEHOLDER.zarr - time_resolution_minutes: 60 diff --git a/src/open_data_pvnet/nwp/dwd.py b/src/open_data_pvnet/nwp/dwd.py index 5957e72..7a755a4 100644 --- a/src/open_data_pvnet/nwp/dwd.py +++ b/src/open_data_pvnet/nwp/dwd.py @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) -CONFIG_PATH = PROJECT_BASE / "src/open_data_pvnet/configs/dwd_data_config.yaml" +CONFIG_PATH = PROJECT_BASE / "src/open_data_pvnet/uk_gfs_configs/dwd_data_config.yaml" class DWDHTMLParser(HTMLParser): def __init__(self): diff --git a/src/open_data_pvnet/nwp/gfs_dataset.py b/src/open_data_pvnet/nwp/gfs_dataset.py index eadb201..2c974ce 100644 --- a/src/open_data_pvnet/nwp/gfs_dataset.py +++ b/src/open_data_pvnet/nwp/gfs_dataset.py @@ -201,7 +201,7 @@ def _normalize_sample(self, dataset: xr.Dataset) -> xr.Dataset: # # Uncomment the block below to test # if __name__ == "__main__": # dataset_path = "s3://ocf-open-data-pvnet/data/gfs.zarr" -# config_path = "src/open_data_pvnet/configs/gfs_data_config.yaml" +# config_path = "src/open_data_pvnet/uk_gfs_configs/gfs_data_config.yaml" # dataset = open_gfs(dataset_path) # dataset = handle_nan_values(dataset, method="fill", fill_value=0.0) # sampler = GFSDataSampler(dataset, config_filename=config_path, start_time="2023-01-01T00:00:00", end_time="2023-01-30T00:00:00") diff --git a/src/open_data_pvnet/nwp/met_office.py b/src/open_data_pvnet/nwp/met_office.py index b77fb43..a3b545c 100644 --- a/src/open_data_pvnet/nwp/met_office.py +++ b/src/open_data_pvnet/nwp/met_office.py @@ -13,8 +13,8 @@ # Define the configuration paths for UK and Global CONFIG_PATHS = { - "uk": PROJECT_BASE / "src/open_data_pvnet/configs/met_office_uk_data_config.yaml", - "global": PROJECT_BASE / "src/open_data_pvnet/configs/met_office_global_data_config.yaml", + "uk": PROJECT_BASE / "src/open_data_pvnet/uk_gfs_configs/met_office_uk_data_config.yaml", + "global": PROJECT_BASE / "src/open_data_pvnet/uk_gfs_configs/met_office_global_data_config.yaml", } diff --git a/src/open_data_pvnet/scripts/save_samples.py b/src/open_data_pvnet/scripts/save_samples.py index 509ee72..6b315ef 100644 --- a/src/open_data_pvnet/scripts/save_samples.py +++ b/src/open_data_pvnet/scripts/save_samples.py @@ -125,7 +125,7 @@ def save_samples_with_dataloader( pbar.close() -@hydra.main(config_path="../configs/PVNet_configs", config_name="config.yaml", version_base="1.2") +@hydra.main(config_path="../uk_gfs_configs/PVNet_configs", config_name="config.yaml", version_base="1.2") def main(config: DictConfig) -> None: """Constructs and saves validation and training samples.""" config_dm = config.datamodule diff --git a/src/open_data_pvnet/configs/PVNet_configs/callbacks/default.yaml b/src/open_data_pvnet/uk_gfs_configs/PVNet_configs/callbacks/default.yaml similarity index 100% rename from src/open_data_pvnet/configs/PVNet_configs/callbacks/default.yaml rename to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/callbacks/default.yaml diff --git a/src/open_data_pvnet/configs/PVNet_configs/config.yaml b/src/open_data_pvnet/uk_gfs_configs/PVNet_configs/config.yaml similarity index 100% rename from src/open_data_pvnet/configs/PVNet_configs/config.yaml rename to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/config.yaml diff --git a/src/open_data_pvnet/configs/PVNet_configs/datamodule/configuration/example_configuration.yaml b/src/open_data_pvnet/uk_gfs_configs/PVNet_configs/datamodule/configuration/example_configuration.yaml similarity index 100% rename from src/open_data_pvnet/configs/PVNet_configs/datamodule/configuration/example_configuration.yaml rename to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/datamodule/configuration/example_configuration.yaml diff --git a/src/open_data_pvnet/configs/PVNet_configs/datamodule/premade_batches.yaml b/src/open_data_pvnet/uk_gfs_configs/PVNet_configs/datamodule/premade_batches.yaml similarity index 100% rename from src/open_data_pvnet/configs/PVNet_configs/datamodule/premade_batches.yaml rename to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/datamodule/premade_batches.yaml diff --git a/src/open_data_pvnet/configs/PVNet_configs/datamodule/streamed_batches.yaml b/src/open_data_pvnet/uk_gfs_configs/PVNet_configs/datamodule/streamed_batches.yaml similarity index 100% rename from src/open_data_pvnet/configs/PVNet_configs/datamodule/streamed_batches.yaml rename to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/datamodule/streamed_batches.yaml diff --git a/src/open_data_pvnet/configs/PVNet_configs/experiment/example_simple.yaml b/src/open_data_pvnet/uk_gfs_configs/PVNet_configs/experiment/example_simple.yaml similarity index 100% rename from src/open_data_pvnet/configs/PVNet_configs/experiment/example_simple.yaml rename to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/experiment/example_simple.yaml diff --git a/src/open_data_pvnet/configs/PVNet_configs/hydra/default.yaml b/src/open_data_pvnet/uk_gfs_configs/PVNet_configs/hydra/default.yaml similarity index 100% rename from src/open_data_pvnet/configs/PVNet_configs/hydra/default.yaml rename to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/hydra/default.yaml diff --git a/src/open_data_pvnet/configs/PVNet_configs/logger/wandb.yaml b/src/open_data_pvnet/uk_gfs_configs/PVNet_configs/logger/wandb.yaml similarity index 100% rename from src/open_data_pvnet/configs/PVNet_configs/logger/wandb.yaml rename to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/logger/wandb.yaml diff --git a/src/open_data_pvnet/configs/PVNet_configs/model/multimodal.yaml b/src/open_data_pvnet/uk_gfs_configs/PVNet_configs/model/multimodal.yaml similarity index 100% rename from src/open_data_pvnet/configs/PVNet_configs/model/multimodal.yaml rename to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/model/multimodal.yaml diff --git a/src/open_data_pvnet/configs/PVNet_configs/readme.md b/src/open_data_pvnet/uk_gfs_configs/PVNet_configs/readme.md similarity index 100% rename from src/open_data_pvnet/configs/PVNet_configs/readme.md rename to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/readme.md diff --git a/src/open_data_pvnet/configs/PVNet_configs/trainer/all_params.yaml b/src/open_data_pvnet/uk_gfs_configs/PVNet_configs/trainer/all_params.yaml similarity index 100% rename from src/open_data_pvnet/configs/PVNet_configs/trainer/all_params.yaml rename to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/trainer/all_params.yaml diff --git a/src/open_data_pvnet/configs/PVNet_configs/trainer/default.yaml b/src/open_data_pvnet/uk_gfs_configs/PVNet_configs/trainer/default.yaml similarity index 100% rename from src/open_data_pvnet/configs/PVNet_configs/trainer/default.yaml rename to src/open_data_pvnet/uk_gfs_configs/PVNet_configs/trainer/default.yaml diff --git a/src/open_data_pvnet/configs/__init.py b/src/open_data_pvnet/uk_gfs_configs/__init.py similarity index 100% rename from src/open_data_pvnet/configs/__init.py rename to src/open_data_pvnet/uk_gfs_configs/__init.py diff --git a/src/open_data_pvnet/configs/gfs_data_config.yaml b/src/open_data_pvnet/uk_gfs_configs/gfs_data_config.yaml similarity index 100% rename from src/open_data_pvnet/configs/gfs_data_config.yaml rename to src/open_data_pvnet/uk_gfs_configs/gfs_data_config.yaml From 1d478709a8eef63861ea40c2657dc2a8e4438ba0 Mon Sep 17 00:00:00 2001 From: Pawan Parida <135092407+zer-art@users.noreply.github.com> Date: Fri, 16 Jan 2026 08:59:13 +0530 Subject: [PATCH 2/2] Revert : restore original config file path for dwd and met_office --- src/open_data_pvnet/nwp/dwd.py | 2 +- src/open_data_pvnet/nwp/met_office.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/open_data_pvnet/nwp/dwd.py b/src/open_data_pvnet/nwp/dwd.py index 7a755a4..5957e72 100644 --- a/src/open_data_pvnet/nwp/dwd.py +++ b/src/open_data_pvnet/nwp/dwd.py @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) -CONFIG_PATH = PROJECT_BASE / "src/open_data_pvnet/uk_gfs_configs/dwd_data_config.yaml" +CONFIG_PATH = PROJECT_BASE / "src/open_data_pvnet/configs/dwd_data_config.yaml" class DWDHTMLParser(HTMLParser): def __init__(self): diff --git a/src/open_data_pvnet/nwp/met_office.py b/src/open_data_pvnet/nwp/met_office.py index a3b545c..b77fb43 100644 --- a/src/open_data_pvnet/nwp/met_office.py +++ b/src/open_data_pvnet/nwp/met_office.py @@ -13,8 +13,8 @@ # Define the configuration paths for UK and Global CONFIG_PATHS = { - "uk": PROJECT_BASE / "src/open_data_pvnet/uk_gfs_configs/met_office_uk_data_config.yaml", - "global": PROJECT_BASE / "src/open_data_pvnet/uk_gfs_configs/met_office_global_data_config.yaml", + "uk": PROJECT_BASE / "src/open_data_pvnet/configs/met_office_uk_data_config.yaml", + "global": PROJECT_BASE / "src/open_data_pvnet/configs/met_office_global_data_config.yaml", }