From 91276e414f4c16a67e121c01a5eb477c4a2f520e Mon Sep 17 00:00:00 2001 From: "jeff.wade" Date: Thu, 5 Feb 2026 16:21:15 -0500 Subject: [PATCH 1/4] Limit GFS downloads by horizon --- .../Global/get_prod_GFS.py | 8 ++- .../forecast_download_base.py | 8 ++- .../medium_range_mem1_config.yml | 63 +++++++++++++++++++ .../medium_range_mem2_config.yml | 63 +++++++++++++++++++ .../medium_range_mem3_config.yml | 63 +++++++++++++++++++ .../medium_range_mem4_config.yml | 63 +++++++++++++++++++ .../medium_range_mem5_config.yml | 63 +++++++++++++++++++ .../medium_range_mem6_config.yml | 63 +++++++++++++++++++ .../medium_range_no_da_config.yml | 63 +++++++++++++++++++ .../forcing_extraction.py | 3 +- 10 files changed, 456 insertions(+), 4 deletions(-) create mode 100644 NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml create mode 100644 NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml create mode 100644 NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml create mode 100644 NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml create mode 100644 NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml create mode 100644 NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml create mode 100644 NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml diff --git a/Forcing_Extraction_Scripts/Global/get_prod_GFS.py b/Forcing_Extraction_Scripts/Global/get_prod_GFS.py index c9b0e020..078fda76 100644 --- a/Forcing_Extraction_Scripts/Global/get_prod_GFS.py +++ b/Forcing_Extraction_Scripts/Global/get_prod_GFS.py @@ -26,7 +26,13 @@ def should_process_hour(self, d_start): def get_download_targets(self, _): hourly = range(1, 121) # 1 through 120 every_3h = range(123, 241, 3) # 123 through 240, step of 3 - return list(hourly) + list(every_3h) + target_hours = list(hourly) + list(every_3h) + + # Filter download targets by input_horizon if specific + if self.input_horizon is not None: + target_hours = [hour for hour in target_hours if hour <= self.input_horizon] + + return target_hours def build_output_dir(self, d_start, _): return os.path.join( diff --git a/Forcing_Extraction_Scripts/forecast_download_base.py b/Forcing_Extraction_Scripts/forecast_download_base.py index 7bccc2a8..2a7b3463 100644 --- a/Forcing_Extraction_Scripts/forecast_download_base.py +++ b/Forcing_Extraction_Scripts/forecast_download_base.py @@ -40,7 +40,7 @@ class ForecastDownloader(ABC): default_cleanback = 240 default_lagback = 6 - def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback_hours, ens_number): + def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback_hours, ens_number, input_horizon=None): """ Initialize downloader with common configuration. @@ -49,6 +49,7 @@ def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback :param lookback_hours: How many hours back to fetch forecasts :param cleanback_hours: How far back to clean old files :param lagback_hours: How many hours to lag before starting to fetch + :param input_horizon: Maximum forecast hour to downlaod (None = download all available timesteps) """ if lookback_hours <= lagback_hours: raise ValueError( @@ -62,6 +63,7 @@ def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback self.cleanback_hours = cleanback_hours self.lagback_hours = lagback_hours self.ens_number = ens_number + self.input_horizon = input_horizon # Current hour, rounded to the top of the hour in UTC self.d_now = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0) @@ -106,6 +108,7 @@ def from_cli_args(cls): parser.add_argument('--cleanBackHours', type=int, default=cls.default_cleanback) parser.add_argument('--lagBackHours', type=int, default=cls.default_lagback) parser.add_argument('--ensNumber', type=int, default=None) + parser.add_argument('--inputHorizon', type=int, default=None) args = parser.parse_args() print(f"{cls.__name__} args:", vars(args)) @@ -116,7 +119,8 @@ def from_cli_args(cls): lookback_hours=args.lookBackHours, cleanback_hours=args.cleanBackHours, lagback_hours=args.lagBackHours, - ens_number=args.ensNumber + ens_number=args.ensNumber, + input_horizon=args.inputHorizon ) def run(self): diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml new file mode 100644 index 00000000..92d47cb9 --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem1" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [1] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem1/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [120] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml new file mode 100644 index 00000000..2793b115 --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem1" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [1] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem1/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [480] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml new file mode 100644 index 00000000..6e340e6c --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem1" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [1] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem1/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [840] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml new file mode 100644 index 00000000..d26d4ab8 --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem1" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [1] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem1/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [1200] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml new file mode 100644 index 00000000..eef861f7 --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem1" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [1] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem1/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [1560] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml new file mode 100644 index 00000000..1fc9c057 --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem1" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [1] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem1/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [1920] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml new file mode 100644 index 00000000..92d47cb9 --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem1" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [1] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem1/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [120] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/forcing_extraction.py b/NextGen_Forcings_Engine_BMI/forcing_extraction.py index 91b62bc5..53305e2e 100644 --- a/NextGen_Forcings_Engine_BMI/forcing_extraction.py +++ b/NextGen_Forcings_Engine_BMI/forcing_extraction.py @@ -141,7 +141,8 @@ def retrieve_forcing(cfg: "ConfigOptions"): lookback_hours=look_back_hours + lag_delt if lag_delt else look_back_hours, cleanback_hours=0, lagback_hours=0, - ens_number=int(ens_number) if ens_number not in ("", None) else None, + ens_number=int(ens_number) if ens_number not in ('', None) else None, + input_horizon=input_horizons[i] if input_horizons[i] > 0 else None, ) # Run the download From cddf76791a5d005e280b750150b36bd4db96b45b Mon Sep 17 00:00:00 2001 From: "jeff.wade" Date: Thu, 5 Feb 2026 16:33:54 -0500 Subject: [PATCH 2/4] Convert input_horizon to hours --- Forcing_Extraction_Scripts/forecast_download_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Forcing_Extraction_Scripts/forecast_download_base.py b/Forcing_Extraction_Scripts/forecast_download_base.py index 2a7b3463..85d87e1d 100644 --- a/Forcing_Extraction_Scripts/forecast_download_base.py +++ b/Forcing_Extraction_Scripts/forecast_download_base.py @@ -63,7 +63,7 @@ def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback self.cleanback_hours = cleanback_hours self.lagback_hours = lagback_hours self.ens_number = ens_number - self.input_horizon = input_horizon + self.input_horizon = int(input_horizon / 60) # Current hour, rounded to the top of the hour in UTC self.d_now = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0) From aeb3f18c8eb7c1bc86a5b56caafa9b5415806082 Mon Sep 17 00:00:00 2001 From: "jeff.wade" Date: Fri, 6 Feb 2026 16:02:42 -0500 Subject: [PATCH 3/4] Update lagged ensemble forcing files and remove GFS debug --- .../config_templates/medium_range_mem1_config.yml | 2 +- .../config_templates/medium_range_mem2_config.yml | 6 +++--- .../config_templates/medium_range_mem3_config.yml | 6 +++--- .../config_templates/medium_range_mem4_config.yml | 6 +++--- .../config_templates/medium_range_mem5_config.yml | 6 +++--- .../config_templates/medium_range_mem6_config.yml | 6 +++--- .../config_templates/medium_range_no_da_config.yml | 2 +- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml index 92d47cb9..32e77bc9 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml @@ -18,7 +18,7 @@ LookBack : -9999 RefcstBDateProc: "202412020000" ForecastFrequency: 60 ForecastShift: 0 -ForecastInputHorizons: [120] +ForecastInputHorizons: [14400] ForecastInputOffsets: [0] Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml index 2793b115..80b48ff0 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml @@ -1,7 +1,7 @@ time_step_seconds: 3600 initial_time: 0 NWM_VERSION: 4.0 -NWM_CONFIG: "medium_range_mem1" +NWM_CONFIG: "medium_range_mem2" InputForcings: [3] InputForcingDirectories: ["{root_dir}/raw_input/GFS"] InputForcingTypes: ["GRIB2"] @@ -9,7 +9,7 @@ InputMandatory: [1] OutputFrequency: 60 SubOutputHour: 120 SubOutFreq: 180 -ScratchDir: "{root_dir}/scratch/med_range_mem1/" +ScratchDir: "{root_dir}/scratch/med_range_mem2/" Output: 0 compressOutput: 0 floatOutput: 0 @@ -18,7 +18,7 @@ LookBack : -9999 RefcstBDateProc: "202412020000" ForecastFrequency: 60 ForecastShift: 0 -ForecastInputHorizons: [480] +ForecastInputHorizons: [12246] ForecastInputOffsets: [0] Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml index 6e340e6c..c7967d83 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml @@ -1,7 +1,7 @@ time_step_seconds: 3600 initial_time: 0 NWM_VERSION: 4.0 -NWM_CONFIG: "medium_range_mem1" +NWM_CONFIG: "medium_range_mem3" InputForcings: [3] InputForcingDirectories: ["{root_dir}/raw_input/GFS"] InputForcingTypes: ["GRIB2"] @@ -9,7 +9,7 @@ InputMandatory: [1] OutputFrequency: 60 SubOutputHour: 120 SubOutFreq: 180 -ScratchDir: "{root_dir}/scratch/med_range_mem1/" +ScratchDir: "{root_dir}/scratch/med_range_mem3/" Output: 0 compressOutput: 0 floatOutput: 0 @@ -18,7 +18,7 @@ LookBack : -9999 RefcstBDateProc: "202412020000" ForecastFrequency: 60 ForecastShift: 0 -ForecastInputHorizons: [840] +ForecastInputHorizons: [12252] ForecastInputOffsets: [0] Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml index d26d4ab8..4934e49e 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml @@ -1,7 +1,7 @@ time_step_seconds: 3600 initial_time: 0 NWM_VERSION: 4.0 -NWM_CONFIG: "medium_range_mem1" +NWM_CONFIG: "medium_range_mem4" InputForcings: [3] InputForcingDirectories: ["{root_dir}/raw_input/GFS"] InputForcingTypes: ["GRIB2"] @@ -9,7 +9,7 @@ InputMandatory: [1] OutputFrequency: 60 SubOutputHour: 120 SubOutFreq: 180 -ScratchDir: "{root_dir}/scratch/med_range_mem1/" +ScratchDir: "{root_dir}/scratch/med_range_mem4/" Output: 0 compressOutput: 0 floatOutput: 0 @@ -18,7 +18,7 @@ LookBack : -9999 RefcstBDateProc: "202412020000" ForecastFrequency: 60 ForecastShift: 0 -ForecastInputHorizons: [1200] +ForecastInputHorizons: [12258] ForecastInputOffsets: [0] Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml index eef861f7..3347c231 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml @@ -1,7 +1,7 @@ time_step_seconds: 3600 initial_time: 0 NWM_VERSION: 4.0 -NWM_CONFIG: "medium_range_mem1" +NWM_CONFIG: "medium_range_mem5" InputForcings: [3] InputForcingDirectories: ["{root_dir}/raw_input/GFS"] InputForcingTypes: ["GRIB2"] @@ -9,7 +9,7 @@ InputMandatory: [1] OutputFrequency: 60 SubOutputHour: 120 SubOutFreq: 180 -ScratchDir: "{root_dir}/scratch/med_range_mem1/" +ScratchDir: "{root_dir}/scratch/med_range_mem5/" Output: 0 compressOutput: 0 floatOutput: 0 @@ -18,7 +18,7 @@ LookBack : -9999 RefcstBDateProc: "202412020000" ForecastFrequency: 60 ForecastShift: 0 -ForecastInputHorizons: [1560] +ForecastInputHorizons: [12264] ForecastInputOffsets: [0] Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml index 1fc9c057..ef8a0a70 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml @@ -1,7 +1,7 @@ time_step_seconds: 3600 initial_time: 0 NWM_VERSION: 4.0 -NWM_CONFIG: "medium_range_mem1" +NWM_CONFIG: "medium_range_mem6" InputForcings: [3] InputForcingDirectories: ["{root_dir}/raw_input/GFS"] InputForcingTypes: ["GRIB2"] @@ -9,7 +9,7 @@ InputMandatory: [1] OutputFrequency: 60 SubOutputHour: 120 SubOutFreq: 180 -ScratchDir: "{root_dir}/scratch/med_range_mem1/" +ScratchDir: "{root_dir}/scratch/med_range_mem6/" Output: 0 compressOutput: 0 floatOutput: 0 @@ -18,7 +18,7 @@ LookBack : -9999 RefcstBDateProc: "202412020000" ForecastFrequency: 60 ForecastShift: 0 -ForecastInputHorizons: [1920] +ForecastInputHorizons: [12270] ForecastInputOffsets: [0] Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml index 92d47cb9..32e77bc9 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml @@ -18,7 +18,7 @@ LookBack : -9999 RefcstBDateProc: "202412020000" ForecastFrequency: 60 ForecastShift: 0 -ForecastInputHorizons: [120] +ForecastInputHorizons: [14400] ForecastInputOffsets: [0] Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" From aee84b0f6a0396c336cbfae88960aecaf89be1e3 Mon Sep 17 00:00:00 2001 From: "jeff.wade" Date: Fri, 6 Mar 2026 12:58:08 -0500 Subject: [PATCH 4/4] Update medium range lagged ensemble templates --- .../config_templates/medium_range_mem1_config.yml | 2 +- .../config_templates/medium_range_mem2_config.yml | 2 +- .../config_templates/medium_range_mem3_config.yml | 2 +- .../config_templates/medium_range_mem4_config.yml | 2 +- .../config_templates/medium_range_mem5_config.yml | 2 +- .../config_templates/medium_range_mem6_config.yml | 2 +- .../config_templates/medium_range_no_da_config.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml index 32e77bc9..7e1507f2 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml @@ -5,7 +5,7 @@ NWM_CONFIG: "medium_range_mem1" InputForcings: [3] InputForcingDirectories: ["{root_dir}/raw_input/GFS"] InputForcingTypes: ["GRIB2"] -InputMandatory: [1] +InputMandatory: [0] OutputFrequency: 60 SubOutputHour: 120 SubOutFreq: 180 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml index 80b48ff0..4410bd60 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml @@ -5,7 +5,7 @@ NWM_CONFIG: "medium_range_mem2" InputForcings: [3] InputForcingDirectories: ["{root_dir}/raw_input/GFS"] InputForcingTypes: ["GRIB2"] -InputMandatory: [1] +InputMandatory: [0] OutputFrequency: 60 SubOutputHour: 120 SubOutFreq: 180 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml index c7967d83..d93e2982 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml @@ -5,7 +5,7 @@ NWM_CONFIG: "medium_range_mem3" InputForcings: [3] InputForcingDirectories: ["{root_dir}/raw_input/GFS"] InputForcingTypes: ["GRIB2"] -InputMandatory: [1] +InputMandatory: [0] OutputFrequency: 60 SubOutputHour: 120 SubOutFreq: 180 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml index 4934e49e..733a5a93 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml @@ -5,7 +5,7 @@ NWM_CONFIG: "medium_range_mem4" InputForcings: [3] InputForcingDirectories: ["{root_dir}/raw_input/GFS"] InputForcingTypes: ["GRIB2"] -InputMandatory: [1] +InputMandatory: [0] OutputFrequency: 60 SubOutputHour: 120 SubOutFreq: 180 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml index 3347c231..1e79c788 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml @@ -5,7 +5,7 @@ NWM_CONFIG: "medium_range_mem5" InputForcings: [3] InputForcingDirectories: ["{root_dir}/raw_input/GFS"] InputForcingTypes: ["GRIB2"] -InputMandatory: [1] +InputMandatory: [0] OutputFrequency: 60 SubOutputHour: 120 SubOutFreq: 180 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml index ef8a0a70..0a63c730 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml @@ -5,7 +5,7 @@ NWM_CONFIG: "medium_range_mem6" InputForcings: [3] InputForcingDirectories: ["{root_dir}/raw_input/GFS"] InputForcingTypes: ["GRIB2"] -InputMandatory: [1] +InputMandatory: [0] OutputFrequency: 60 SubOutputHour: 120 SubOutFreq: 180 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml index 32e77bc9..7e1507f2 100644 --- a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml @@ -5,7 +5,7 @@ NWM_CONFIG: "medium_range_mem1" InputForcings: [3] InputForcingDirectories: ["{root_dir}/raw_input/GFS"] InputForcingTypes: ["GRIB2"] -InputMandatory: [1] +InputMandatory: [0] OutputFrequency: 60 SubOutputHour: 120 SubOutFreq: 180