diff --git a/Forcing_Extraction_Scripts/Global/get_prod_GFS.py b/Forcing_Extraction_Scripts/Global/get_prod_GFS.py index c9b0e020..078fda76 100644 --- a/Forcing_Extraction_Scripts/Global/get_prod_GFS.py +++ b/Forcing_Extraction_Scripts/Global/get_prod_GFS.py @@ -26,7 +26,13 @@ def should_process_hour(self, d_start): def get_download_targets(self, _): hourly = range(1, 121) # 1 through 120 every_3h = range(123, 241, 3) # 123 through 240, step of 3 - return list(hourly) + list(every_3h) + target_hours = list(hourly) + list(every_3h) + + # Filter download targets by input_horizon if specific + if self.input_horizon is not None: + target_hours = [hour for hour in target_hours if hour <= self.input_horizon] + + return target_hours def build_output_dir(self, d_start, _): return os.path.join( diff --git a/Forcing_Extraction_Scripts/forecast_download_base.py b/Forcing_Extraction_Scripts/forecast_download_base.py index 7bccc2a8..85d87e1d 100644 --- a/Forcing_Extraction_Scripts/forecast_download_base.py +++ b/Forcing_Extraction_Scripts/forecast_download_base.py @@ -40,7 +40,7 @@ class ForecastDownloader(ABC): default_cleanback = 240 default_lagback = 6 - def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback_hours, ens_number): + def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback_hours, ens_number, input_horizon=None): """ Initialize downloader with common configuration. @@ -49,6 +49,7 @@ def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback :param lookback_hours: How many hours back to fetch forecasts :param cleanback_hours: How far back to clean old files :param lagback_hours: How many hours to lag before starting to fetch + :param input_horizon: Maximum forecast hour to downlaod (None = download all available timesteps) """ if lookback_hours <= lagback_hours: raise ValueError( @@ -62,6 +63,7 @@ def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback self.cleanback_hours = cleanback_hours self.lagback_hours = lagback_hours self.ens_number = ens_number + self.input_horizon = int(input_horizon / 60) # Current hour, rounded to the top of the hour in UTC self.d_now = datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0) @@ -106,6 +108,7 @@ def from_cli_args(cls): parser.add_argument('--cleanBackHours', type=int, default=cls.default_cleanback) parser.add_argument('--lagBackHours', type=int, default=cls.default_lagback) parser.add_argument('--ensNumber', type=int, default=None) + parser.add_argument('--inputHorizon', type=int, default=None) args = parser.parse_args() print(f"{cls.__name__} args:", vars(args)) @@ -116,7 +119,8 @@ def from_cli_args(cls): lookback_hours=args.lookBackHours, cleanback_hours=args.cleanBackHours, lagback_hours=args.lagBackHours, - ens_number=args.ensNumber + ens_number=args.ensNumber, + input_horizon=args.inputHorizon ) def run(self): diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml new file mode 100644 index 00000000..7e1507f2 --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem1_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem1" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [0] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem1/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [14400] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml new file mode 100644 index 00000000..4410bd60 --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem2_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem2" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [0] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem2/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [12246] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml new file mode 100644 index 00000000..d93e2982 --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem3_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem3" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [0] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem3/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [12252] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml new file mode 100644 index 00000000..733a5a93 --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem4_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem4" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [0] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem4/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [12258] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml new file mode 100644 index 00000000..1e79c788 --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem5_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem5" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [0] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem5/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [12264] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml new file mode 100644 index 00000000..0a63c730 --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_mem6_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem6" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [0] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem6/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [12270] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml new file mode 100644 index 00000000..7e1507f2 --- /dev/null +++ b/NextGen_Forcings_Engine_BMI/BMI_NextGen_Configs/config_templates/medium_range_no_da_config.yml @@ -0,0 +1,63 @@ +time_step_seconds: 3600 +initial_time: 0 +NWM_VERSION: 4.0 +NWM_CONFIG: "medium_range_mem1" +InputForcings: [3] +InputForcingDirectories: ["{root_dir}/raw_input/GFS"] +InputForcingTypes: ["GRIB2"] +InputMandatory: [0] +OutputFrequency: 60 +SubOutputHour: 120 +SubOutFreq: 180 +ScratchDir: "{root_dir}/scratch/med_range_mem1/" +Output: 0 +compressOutput: 0 +floatOutput: 0 +AnAFlag: 0 +LookBack : -9999 +RefcstBDateProc: "202412020000" +ForecastFrequency: 60 +ForecastShift: 0 +ForecastInputHorizons: [14400] +ForecastInputOffsets: [0] +Geopackage: "{root_dir}/gpkg/conus/{gage}.gpkg" +GeogridIn: "{root_dir}/esmf_mesh/{gage}_ESMF_Mesh.nc" +SpatialMetaIn: "" +GRID_TYPE: "hydrofabric" +NodeCoords: 'nodeCoords' +ElemCoords: 'centerCoords' +ElemConn: 'elementConn' +ElemID: 'element_id' +NumElemConn: 'numElementConn' +HGTVAR: 'Element_Elevation' +SLOPE: 'Element_Slope' +SLOPE_AZIMUTH: 'Element_Slope_Azmuith' +IgnoredBorderWidths: [0] +RegridOpt: [1] +ForcingTemporalInterpolation: [0] +TemperatureBiasCorrection: [0] +PressureBiasCorrection: [0] +HumidityBiasCorrection: [0] +WindBiasCorrection: [0] +SwBiasCorrection: [0] +LwBiasCorrection: [0] +PrecipBiasCorrection: [0] +TemperatureDownscaling: [0] +ShortwaveDownscaling: [0] +PressureDownscaling: [0] +PrecipDownscaling: [0] +HumidityDownscaling: [0] +DownscalingParamDirs: ["{root_dir}"] +SuppPcp: [] +SuppPcpForcingTypes: [] +SuppPcpDirectories: [] +SuppPcpParamDir: '{root_dir}' +RegridOptSuppPcp: [] +SuppPcpTemporalInterpolation: [] +SuppPcpInputOffsets: [] +SuppPcpMandatory: [] +RqiMethod: 0 +RqiThreshold: 0.9 +cfsEnsNumber: '' +custom_input_fcst_freq: [] +includeLQFrac: 1 diff --git a/NextGen_Forcings_Engine_BMI/forcing_extraction.py b/NextGen_Forcings_Engine_BMI/forcing_extraction.py index 91b62bc5..53305e2e 100644 --- a/NextGen_Forcings_Engine_BMI/forcing_extraction.py +++ b/NextGen_Forcings_Engine_BMI/forcing_extraction.py @@ -141,7 +141,8 @@ def retrieve_forcing(cfg: "ConfigOptions"): lookback_hours=look_back_hours + lag_delt if lag_delt else look_back_hours, cleanback_hours=0, lagback_hours=0, - ens_number=int(ens_number) if ens_number not in ("", None) else None, + ens_number=int(ens_number) if ens_number not in ('', None) else None, + input_horizon=input_horizons[i] if input_horizons[i] > 0 else None, ) # Run the download