Add forcing engine support for lagged ensemble members

Forcing_Extraction_Scripts/Global/get_prod_GFS.py

@@ -26,7 +26,13 @@ def should_process_hour(self, d_start):
     def get_download_targets(self, _):
         hourly = range(1, 121)  # 1 through 120
         every_3h = range(123, 241, 3)  # 123 through 240, step of 3
-        return list(hourly) + list(every_3h)
+        target_hours = list(hourly) + list(every_3h)
+
+        # Filter download targets by input_horizon if specific
+        if self.input_horizon is not None:
+            target_hours = [hour for hour in target_hours if hour <= self.input_horizon]
+
+        return target_hours
 
     def build_output_dir(self, d_start, _):
         return os.path.join(

Forcing_Extraction_Scripts/forecast_download_base.py

@@ -40,7 +40,7 @@ class ForecastDownloader(ABC):
     default_cleanback = 240
     default_lagback = 6
 
-    def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback_hours, ens_number):
+    def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback_hours, ens_number, input_horizon=None):
         """
         Initialize downloader with common configuration.
 
@@ -49,6 +49,7 @@ def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback
         :param lookback_hours: How many hours back to fetch forecasts
         :param cleanback_hours: How far back to clean old files
         :param lagback_hours: How many hours to lag before starting to fetch
+        :param input_horizon: Maximum forecast hour to downlaod (None = download all available timesteps)
         """
         if lookback_hours <= lagback_hours:
             raise ValueError(
@@ -62,6 +63,7 @@ def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback
         self.cleanback_hours = cleanback_hours
         self.lagback_hours = lagback_hours
         self.ens_number = ens_number
+        self.input_horizon = int(input_horizon / 60)
 
         # Current hour, rounded to the top of the hour in UTC
         self.d_now = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0)
@@ -106,6 +108,7 @@ def from_cli_args(cls):
         parser.add_argument('--cleanBackHours', type=int, default=cls.default_cleanback)
         parser.add_argument('--lagBackHours', type=int, default=cls.default_lagback)
         parser.add_argument('--ensNumber', type=int, default=None)
+        parser.add_argument('--inputHorizon', type=int, default=None)
         args = parser.parse_args()
 
         print(f"{cls.__name__} args:", vars(args))
@@ -116,7 +119,8 @@ def from_cli_args(cls):
             lookback_hours=args.lookBackHours,
             cleanback_hours=args.cleanBackHours,
             lagback_hours=args.lagBackHours,
-            ens_number=args.ensNumber
+            ens_number=args.ensNumber,
+            input_horizon=args.inputHorizon
         )
 
     def run(self):

NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml

@@ -0,0 +1,63 @@
+time_step_seconds: 3600
+initial_time: 0
+NWM_VERSION: 4.0
+NWM_CONFIG: "medium_range_mem1"
+InputForcings: [3] 
+InputForcingDirectories: ["{root_dir}/raw_input/GFS"]
+InputForcingTypes: ["GRIB2"]
+InputMandatory: [0]
+OutputFrequency: 60
+SubOutputHour: 120
+SubOutFreq: 180
+ScratchDir: "{root_dir}/scratch/med_range_mem1/"
+Output: 0
+compressOutput: 0
+floatOutput: 0
+AnAFlag: 0
+LookBack : -9999
+RefcstBDateProc: "202412020000"
+ForecastFrequency: 60
+ForecastShift: 0
+ForecastInputHorizons: [14400]
+ForecastInputOffsets: [0]
+Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg"
+GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc"
+SpatialMetaIn: ""
+GRID_TYPE: "hydrofabric"
+NodeCoords: 'nodeCoords'
+ElemCoords: 'centerCoords'
+ElemConn: 'elementConn'
+ElemID: 'element_id'
+NumElemConn: 'numElementConn'
+HGTVAR: 'Element_Elevation'
+SLOPE: 'Element_Slope'
+SLOPE_AZIMUTH: 'Element_Slope_Azmuith'
+IgnoredBorderWidths: [0]
+RegridOpt: [1]
+ForcingTemporalInterpolation: [0]
+TemperatureBiasCorrection: [0]
+PressureBiasCorrection: [0]
+HumidityBiasCorrection: [0]
+WindBiasCorrection: [0]
+SwBiasCorrection: [0]
+LwBiasCorrection: [0]
+PrecipBiasCorrection: [0]
+TemperatureDownscaling: [0]
+ShortwaveDownscaling: [0]
+PressureDownscaling: [0]
+PrecipDownscaling: [0]
+HumidityDownscaling: [0]
+DownscalingParamDirs: ["{root_dir}"]
+SuppPcp: []
+SuppPcpForcingTypes: []
+SuppPcpDirectories: []
+SuppPcpParamDir: '{root_dir}'
+RegridOptSuppPcp: []
+SuppPcpTemporalInterpolation: []
+SuppPcpInputOffsets: []
+SuppPcpMandatory: []
+RqiMethod: 0
+RqiThreshold: 0.9
+cfsEnsNumber: ''
+custom_input_fcst_freq: []
+includeLQFrac: 1

NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml

@@ -0,0 +1,63 @@
+time_step_seconds: 3600
+initial_time: 0
+NWM_VERSION: 4.0
+NWM_CONFIG: "medium_range_mem2"
+InputForcings: [3] 
+InputForcingDirectories: ["{root_dir}/raw_input/GFS"]
+InputForcingTypes: ["GRIB2"]
+InputMandatory: [0]
+OutputFrequency: 60
+SubOutputHour: 120
+SubOutFreq: 180
+ScratchDir: "{root_dir}/scratch/med_range_mem2/"
+Output: 0
+compressOutput: 0
+floatOutput: 0
+AnAFlag: 0
+LookBack : -9999
+RefcstBDateProc: "202412020000"
+ForecastFrequency: 60
+ForecastShift: 0
+ForecastInputHorizons: [12246]
+ForecastInputOffsets: [0]
+Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg"
+GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc"
+SpatialMetaIn: ""
+GRID_TYPE: "hydrofabric"
+NodeCoords: 'nodeCoords'
+ElemCoords: 'centerCoords'
+ElemConn: 'elementConn'
+ElemID: 'element_id'
+NumElemConn: 'numElementConn'
+HGTVAR: 'Element_Elevation'
+SLOPE: 'Element_Slope'
+SLOPE_AZIMUTH: 'Element_Slope_Azmuith'
+IgnoredBorderWidths: [0]
+RegridOpt: [1]
+ForcingTemporalInterpolation: [0]
+TemperatureBiasCorrection: [0]
+PressureBiasCorrection: [0]
+HumidityBiasCorrection: [0]
+WindBiasCorrection: [0]
+SwBiasCorrection: [0]
+LwBiasCorrection: [0]
+PrecipBiasCorrection: [0]
+TemperatureDownscaling: [0]
+ShortwaveDownscaling: [0]
+PressureDownscaling: [0]
+PrecipDownscaling: [0]
+HumidityDownscaling: [0]
+DownscalingParamDirs: ["{root_dir}"]
+SuppPcp: []
+SuppPcpForcingTypes: []
+SuppPcpDirectories: []
+SuppPcpParamDir: '{root_dir}'
+RegridOptSuppPcp: []
+SuppPcpTemporalInterpolation: []
+SuppPcpInputOffsets: []
+SuppPcpMandatory: []
+RqiMethod: 0
+RqiThreshold: 0.9
+cfsEnsNumber: ''
+custom_input_fcst_freq: []
+includeLQFrac: 1

NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml

@@ -0,0 +1,63 @@
+time_step_seconds: 3600
+initial_time: 0
+NWM_VERSION: 4.0
+NWM_CONFIG: "medium_range_mem3"
+InputForcings: [3] 
+InputForcingDirectories: ["{root_dir}/raw_input/GFS"]
+InputForcingTypes: ["GRIB2"]
+InputMandatory: [0]
+OutputFrequency: 60
+SubOutputHour: 120
+SubOutFreq: 180
+ScratchDir: "{root_dir}/scratch/med_range_mem3/"
+Output: 0
+compressOutput: 0
+floatOutput: 0
+AnAFlag: 0
+LookBack : -9999
+RefcstBDateProc: "202412020000"
+ForecastFrequency: 60
+ForecastShift: 0
+ForecastInputHorizons: [12252]
+ForecastInputOffsets: [0]
+Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg"
+GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc"
+SpatialMetaIn: ""
+GRID_TYPE: "hydrofabric"
+NodeCoords: 'nodeCoords'
+ElemCoords: 'centerCoords'
+ElemConn: 'elementConn'
+ElemID: 'element_id'
+NumElemConn: 'numElementConn'
+HGTVAR: 'Element_Elevation'
+SLOPE: 'Element_Slope'
+SLOPE_AZIMUTH: 'Element_Slope_Azmuith'
+IgnoredBorderWidths: [0]
+RegridOpt: [1]
+ForcingTemporalInterpolation: [0]
+TemperatureBiasCorrection: [0]
+PressureBiasCorrection: [0]
+HumidityBiasCorrection: [0]
+WindBiasCorrection: [0]
+SwBiasCorrection: [0]
+LwBiasCorrection: [0]
+PrecipBiasCorrection: [0]
+TemperatureDownscaling: [0]
+ShortwaveDownscaling: [0]
+PressureDownscaling: [0]
+PrecipDownscaling: [0]
+HumidityDownscaling: [0]
+DownscalingParamDirs: ["{root_dir}"]
+SuppPcp: []
+SuppPcpForcingTypes: []
+SuppPcpDirectories: []
+SuppPcpParamDir: '{root_dir}'
+RegridOptSuppPcp: []
+SuppPcpTemporalInterpolation: []
+SuppPcpInputOffsets: []
+SuppPcpMandatory: []
+RqiMethod: 0
+RqiThreshold: 0.9
+cfsEnsNumber: ''
+custom_input_fcst_freq: []
+includeLQFrac: 1

NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml

@@ -0,0 +1,63 @@
+time_step_seconds: 3600
+initial_time: 0
+NWM_VERSION: 4.0
+NWM_CONFIG: "medium_range_mem4"
+InputForcings: [3] 
+InputForcingDirectories: ["{root_dir}/raw_input/GFS"]
+InputForcingTypes: ["GRIB2"]
+InputMandatory: [0]
+OutputFrequency: 60
+SubOutputHour: 120
+SubOutFreq: 180
+ScratchDir: "{root_dir}/scratch/med_range_mem4/"
+Output: 0
+compressOutput: 0
+floatOutput: 0
+AnAFlag: 0
+LookBack : -9999
+RefcstBDateProc: "202412020000"
+ForecastFrequency: 60
+ForecastShift: 0
+ForecastInputHorizons: [12258]
+ForecastInputOffsets: [0]
+Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg"
+GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc"
+SpatialMetaIn: ""
+GRID_TYPE: "hydrofabric"
+NodeCoords: 'nodeCoords'
+ElemCoords: 'centerCoords'
+ElemConn: 'elementConn'
+ElemID: 'element_id'
+NumElemConn: 'numElementConn'
+HGTVAR: 'Element_Elevation'
+SLOPE: 'Element_Slope'
+SLOPE_AZIMUTH: 'Element_Slope_Azmuith'
+IgnoredBorderWidths: [0]
+RegridOpt: [1]
+ForcingTemporalInterpolation: [0]
+TemperatureBiasCorrection: [0]
+PressureBiasCorrection: [0]
+HumidityBiasCorrection: [0]
+WindBiasCorrection: [0]
+SwBiasCorrection: [0]
+LwBiasCorrection: [0]
+PrecipBiasCorrection: [0]
+TemperatureDownscaling: [0]
+ShortwaveDownscaling: [0]
+PressureDownscaling: [0]
+PrecipDownscaling: [0]
+HumidityDownscaling: [0]
+DownscalingParamDirs: ["{root_dir}"]
+SuppPcp: []
+SuppPcpForcingTypes: []
+SuppPcpDirectories: []
+SuppPcpParamDir: '{root_dir}'
+RegridOptSuppPcp: []
+SuppPcpTemporalInterpolation: []
+SuppPcpInputOffsets: []
+SuppPcpMandatory: []
+RqiMethod: 0
+RqiThreshold: 0.9
+cfsEnsNumber: ''
+custom_input_fcst_freq: []
+includeLQFrac: 1

NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml

@@ -0,0 +1,63 @@
+time_step_seconds: 3600
+initial_time: 0
+NWM_VERSION: 4.0
+NWM_CONFIG: "medium_range_mem5"
+InputForcings: [3] 
+InputForcingDirectories: ["{root_dir}/raw_input/GFS"]
+InputForcingTypes: ["GRIB2"]
+InputMandatory: [0]
+OutputFrequency: 60
+SubOutputHour: 120
+SubOutFreq: 180
+ScratchDir: "{root_dir}/scratch/med_range_mem5/"
+Output: 0
+compressOutput: 0
+floatOutput: 0
+AnAFlag: 0
+LookBack : -9999
+RefcstBDateProc: "202412020000"
+ForecastFrequency: 60
+ForecastShift: 0
+ForecastInputHorizons: [12264]
+ForecastInputOffsets: [0]
+Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg"
+GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc"
+SpatialMetaIn: ""
+GRID_TYPE: "hydrofabric"
+NodeCoords: 'nodeCoords'
+ElemCoords: 'centerCoords'
+ElemConn: 'elementConn'
+ElemID: 'element_id'
+NumElemConn: 'numElementConn'
+HGTVAR: 'Element_Elevation'
+SLOPE: 'Element_Slope'
+SLOPE_AZIMUTH: 'Element_Slope_Azmuith'
+IgnoredBorderWidths: [0]
+RegridOpt: [1]
+ForcingTemporalInterpolation: [0]
+TemperatureBiasCorrection: [0]
+PressureBiasCorrection: [0]
+HumidityBiasCorrection: [0]
+WindBiasCorrection: [0]
+SwBiasCorrection: [0]
+LwBiasCorrection: [0]
+PrecipBiasCorrection: [0]
+TemperatureDownscaling: [0]
+ShortwaveDownscaling: [0]
+PressureDownscaling: [0]
+PrecipDownscaling: [0]
+HumidityDownscaling: [0]
+DownscalingParamDirs: ["{root_dir}"]
+SuppPcp: []
+SuppPcpForcingTypes: []
+SuppPcpDirectories: []
+SuppPcpParamDir: '{root_dir}'
+RegridOptSuppPcp: []
+SuppPcpTemporalInterpolation: []
+SuppPcpInputOffsets: []
+SuppPcpMandatory: []
+RqiMethod: 0
+RqiThreshold: 0.9
+cfsEnsNumber: ''
+custom_input_fcst_freq: []
+includeLQFrac: 1