From 369291297084b6320c66ed99bc356c891f40ee5a Mon Sep 17 00:00:00 2001 From: "Carolyn.Maynard" Date: Mon, 16 Mar 2026 17:43:09 -0700 Subject: [PATCH 1/3] Updates to use the nwm-ewts libraries --- .../forecast_download_base.py | 21 +- .../NextGen_Forcings_Engine/bmi_grid.py | 7 +- .../NextGen_Forcings_Engine/bmi_model.py | 17 +- .../NextGen_Forcings_Engine/core/config.py | 6 +- .../core/err_handler.py | 180 ++---------------- .../core/forcingInputMod.py | 7 +- .../core/forecastMod.py | 39 +--- .../NextGen_Forcings_Engine/core/geoMod.py | 9 +- .../NextGen_Forcings_Engine/core/ioMod.py | 7 +- .../NextGen_Forcings_Engine/core/regrid.py | 7 +- .../core/time_handling.py | 7 +- .../historical_forcing.py | 8 +- .../NextGen_Forcings_Engine/model.py | 29 +-- nextgen_forcings_ewts/pyproject.toml | 8 - .../src/nextgen_forcings_ewts/__init__.py | 34 ---- .../src/nextgen_forcings_ewts/config.py | 127 ------------ .../src/nextgen_forcings_ewts/constants.py | 40 ---- .../src/nextgen_forcings_ewts/formatter.py | 73 ------- .../src/nextgen_forcings_ewts/helper.py | 32 ---- .../src/nextgen_forcings_ewts/paths.py | 117 ------------ pyproject.toml | 3 +- tests/nextgen_forcings_ewts/conftest.py | 25 --- tests/nextgen_forcings_ewts/test_config.py | 81 -------- tests/nextgen_forcings_ewts/test_constants.py | 10 - tests/nextgen_forcings_ewts/test_formatter.py | 66 ------- tests/nextgen_forcings_ewts/test_paths.py | 115 ----------- 26 files changed, 69 insertions(+), 1006 deletions(-) delete mode 100644 nextgen_forcings_ewts/pyproject.toml delete mode 100644 nextgen_forcings_ewts/src/nextgen_forcings_ewts/__init__.py delete mode 100644 nextgen_forcings_ewts/src/nextgen_forcings_ewts/config.py delete mode 100644 nextgen_forcings_ewts/src/nextgen_forcings_ewts/constants.py delete mode 100644 nextgen_forcings_ewts/src/nextgen_forcings_ewts/formatter.py delete mode 100644 nextgen_forcings_ewts/src/nextgen_forcings_ewts/helper.py delete mode 100644 nextgen_forcings_ewts/src/nextgen_forcings_ewts/paths.py delete mode 100644 tests/nextgen_forcings_ewts/conftest.py delete mode 100644 tests/nextgen_forcings_ewts/test_config.py delete mode 100644 tests/nextgen_forcings_ewts/test_constants.py delete mode 100644 tests/nextgen_forcings_ewts/test_formatter.py delete mode 100644 tests/nextgen_forcings_ewts/test_paths.py diff --git a/Forcing_Extraction_Scripts/forecast_download_base.py b/Forcing_Extraction_Scripts/forecast_download_base.py index 7bccc2a8..65df13c3 100644 --- a/Forcing_Extraction_Scripts/forecast_download_base.py +++ b/Forcing_Extraction_Scripts/forecast_download_base.py @@ -11,14 +11,9 @@ import requests from bs4 import BeautifulSoup -from nextgen_forcings_ewts import MODULE_NAME - -LOG = logging.getLogger(MODULE_NAME) -if not LOG.handlers: - # No handlers attached — fallback to default root logger - logging.basicConfig() - LOG = logging.getLogger() - +# Use the Error, Warning, and Trapping System Package for logging +import ewts +LOG = ewts.get_logger(ewts.FORCING_ID) class ForecastDownloader(ABC): """ @@ -50,6 +45,16 @@ def __init__(self, out_dir, start_time, lookback_hours, cleanback_hours, lagback :param cleanback_hours: How far back to clean old files :param lagback_hours: How many hours to lag before starting to fetch """ + + global LOG + if hasattr(LOG, "bind"): + # This is required prior to the first log message for the ewts package + LOG.bind() + else: + # Fallback to default root logger + logging.basicConfig() + LOG = logging.getLogger() + if lookback_hours <= lagback_hours: raise ValueError( f"Invalid configuration: lookback_hours ({lookback_hours}) must be greater than " diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_grid.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_grid.py index 7a36c673..4ac56a2b 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_grid.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_grid.py @@ -13,11 +13,10 @@ if TYPE_CHECKING: from numpy.typing import NDArray -import logging -from nextgen_forcings_ewts import MODULE_NAME - -LOG = logging.getLogger(MODULE_NAME) +# Use the Error, Warning, and Trapping System Package for logging +import ewts +LOG = ewts.get_logger(ewts.FORCING_ID) _error_on_grid_type: bool = False diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py index 4739e19b..8394fd58 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/bmi_model.py @@ -53,20 +53,14 @@ from numpy.typing import NDArray +# Use the Error, Warning, and Trapping System Package for logging +import ewts +LOG = ewts.get_logger(ewts.FORCING_ID) + # If less than 0, then ESMF.__version__ is greater than 8.7.0 if ESMF.version_compare("8.7.0", ESMF.__version__) < 0: manager = ESMF.api.esmpymanager.Manager(endFlag=ESMF.constants.EndAction.KEEP_MPI) -import logging - -from nextgen_forcings_ewts import MODULE_NAME, configure_logging - -configure_logging() - - -LOG = logging.getLogger(MODULE_NAME) - - class UnknownBMIVariable(RuntimeError): """Custom exception raised when an unknown BMI variable is encountered.""" @@ -182,6 +176,9 @@ def initialize(self, config_file: str, output_path: str | None = None) -> None: :param config_file: The path to the configuration file for model initialization. :raises RuntimeError: If the configuration file is invalid or missing. """ + # This is required prior to the first log message. + LOG.bind() + LOG.info("---------------------------") LOG.info(f"BMI Forcing Engine initialized with {config_file}") diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py index 4eefcb6a..da66627b 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/config.py @@ -14,12 +14,12 @@ from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.time_handling import ( calculate_lookback_window, ) -from nextgen_forcings_ewts import MODULE_NAME - from . import mpi_utils -LOG = logging.getLogger(MODULE_NAME) +# Use the Error, Warning, and Trapping System Package for logging +import ewts +LOG = ewts.get_logger(ewts.FORCING_ID) FORCE_COUNT = 27 diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/err_handler.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/err_handler.py index c69c0553..fcbfc9db 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/err_handler.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/err_handler.py @@ -2,16 +2,14 @@ import os import sys import traceback -from logging import FileHandler import numpy as np from mpi4py import MPI from scipy import spatial -from nextgen_forcings_ewts import MODULE_NAME - -LOG = logging.getLogger(MODULE_NAME) -log_name = MODULE_NAME +# Use the Error, Warning, and Trapping System Package for logging +import ewts +LOG = ewts.get_logger(ewts.FORCING_ID) def err_out_screen(err_msg: str, exc: BaseException | None = None): @@ -123,66 +121,6 @@ def check_program_status( # their error flag, since non-0 ranks do not block on reduce(). # MpiConfig.comm.barrier() - -def init_log(ConfigOptions, MpiConfig): - """Initialize the per-cycle log file once on rank 0. - - Initialize the per‑cycle log file once on rank 0. - We only want a single log file per cycle—not one per catchment—so we check - existing FileHandlers to avoid opening multiple handlers for the same file. - """ - # Only the master rank sets up logging - if MpiConfig.rank != 0: - return - - global log_name - global LOG - - # Check for ngen Error and Warning Trapping System named logger - logger = logging.getLogger(MODULE_NAME) - - # checking whether the logger object has an attribute named _initialized, - # and if it does, whether its value is True. If the attribute doesn't exist, - # it defaults to False. - if getattr(logger, "_initialized", False): - for handler in logger.handlers: - if isinstance(handler, logging.FileHandler): - ConfigOptions.logFile = handler.baseFilename - break - log_name = MODULE_NAME - LOG = logger - return # logger already initialized, nothing else to do - - log_name = "logForcing" - filename = ConfigOptions.logFile - - try: - logger = logging.getLogger(log_name) - - # If a FileHandler for this filename is already attached, skip (prevents one log per catchment) - for handler in logger.handlers: - if ( - isinstance(handler, FileHandler) - and getattr(handler, "baseFilename", None) == filename - ): - LOG = logger - return - - # Otherwise, create and attach a new FileHandler - formatter = logging.Formatter( - "[%(asctime)s]: %(levelname)s - %(message)s", datefmt="%m/%d %H:%M:%S" - ) - file_handler = FileHandler(filename, mode="a") - file_handler.setFormatter(formatter) - logger.addHandler(file_handler) - logger.setLevel(logging.INFO) - LOG = logger - - except Exception as e: - ConfigOptions.errMsg = f"Unable to initialize log file '{filename}': {e}" - err_out_screen_para(ConfigOptions.errMsg, MpiConfig) - - def err_out(ConfigOptions): """Error out after an error message has been logged for a forecast cycle. @@ -191,17 +129,12 @@ def err_out(ConfigOptions): :param ConfigOptions: :return: """ - if not LOG.hasHandlers(): - ConfigOptions.errMsg = ( - "Unable to obtain a logger object for: " + ConfigOptions.logFile - ) - raise Exception() try: LOG.error(ConfigOptions.errMsg) except Exception: ConfigOptions.errMsg = ( - "Unable to write error message to: " + ConfigOptions.logFile + "Unable to write error message" ) raise Exception() MPI.Finalize() @@ -217,13 +150,11 @@ def log_error(ConfigOptions, MpiConfig, msg: str = None): :return: """ if msg is not None: - ConfigOptions.errMsg = msg - - if not LOG.hasHandlers(): - ConfigOptions.errMsg = ( - "Unable to obtain a logger object for: " + ConfigOptions.logFile - ) - raise Exception() + if not isinstance(msg, str): + raise TypeError( + f"Expected type str or NoneType for msg, got type: {type(msg)}" + ) + ConfigOptions.statusMsg = msg try: LOG.error("RANK: " + str(MpiConfig.rank) + " - " + ConfigOptions.errMsg) @@ -232,8 +163,6 @@ def log_error(ConfigOptions, MpiConfig, msg: str = None): ( "Unable to write ERROR message on RANK: " + str(MpiConfig.rank) - + " for log file: " - + ConfigOptions.logFile ), MpiConfig, ) @@ -248,13 +177,11 @@ def log_critical(ConfigOptions, MpiConfig, msg: str = None): :return: """ if msg is not None: - ConfigOptions.errMsg = msg - - if not LOG.hasHandlers(): - ConfigOptions.errMsg = ( - "Unable to obtain a logger object for: " + ConfigOptions.logFile - ) - raise Exception() + if not isinstance(msg, str): + raise TypeError( + f"Expected type str or NoneType for msg, got type: {type(msg)}" + ) + ConfigOptions.statusMsg = msg try: LOG.critical("RANK: " + str(MpiConfig.rank) + " - " + ConfigOptions.errMsg) @@ -263,8 +190,6 @@ def log_critical(ConfigOptions, MpiConfig, msg: str = None): ( "Unable to write CRITICAL message on RANK: " + str(MpiConfig.rank) - + " for log file: " - + ConfigOptions.logFile ), MpiConfig, ) @@ -283,14 +208,12 @@ def log_warning(ConfigOptions, MpiConfig, msg: str = None): :return: """ if msg is not None: + if not isinstance(msg, str): + raise TypeError( + f"Expected type str or NoneType for msg, got type: {type(msg)}" + ) ConfigOptions.statusMsg = msg - if not LOG.hasHandlers(): - ConfigOptions.errMsg = ( - "Unable to obtain a logger object for: " + ConfigOptions.logFile - ) - raise Exception() - try: LOG.warning("RANK: " + str(MpiConfig.rank) + " - " + ConfigOptions.statusMsg) except Exception: @@ -298,8 +221,6 @@ def log_warning(ConfigOptions, MpiConfig, msg: str = None): ( "Unable to write WARNING message on RANK: " + str(MpiConfig.rank) - + " for log file: " - + ConfigOptions.logFile ), MpiConfig, ) @@ -314,6 +235,7 @@ def log_msg(ConfigOptions, MpiConfig, debug: bool = False, msg: str = None): """ if not isinstance(debug, bool): raise TypeError(f"Expected type bool for debug, got type: {type(debug)}") + if msg is not None: if not isinstance(msg, str): raise TypeError( @@ -321,12 +243,6 @@ def log_msg(ConfigOptions, MpiConfig, debug: bool = False, msg: str = None): ) ConfigOptions.statusMsg = msg - if not LOG.hasHandlers(): - ConfigOptions.errMsg = ( - "log_msg: Unable to obtain a logger object for: " + ConfigOptions.logFile - ) - raise Exception() - try: if debug: LOG.debug("RANK: " + str(MpiConfig.rank) + " - " + ConfigOptions.statusMsg) @@ -337,68 +253,10 @@ def log_msg(ConfigOptions, MpiConfig, debug: bool = False, msg: str = None): ( "Unable to write log_msg message on RANK: " + str(MpiConfig.rank) - + " for log file: " - + ConfigOptions.logFile - ), - MpiConfig, - ) - - -def close_log(ConfigOptions, MpiConfig): - """Close the log file. - - Function for closing a log file. - :param ConfigOptions: - :return: - """ - # Only close if we have an open handler - if getattr(ConfigOptions, "logHandle", None) is None: - return - - if log_name == MODULE_NAME: - return - - try: - logObj = logging.getLogger(log_name) - except Exception: - err_out_screen_para( - ( - "Unable to obtain logger object on RANK: " - + str(MpiConfig.rank) - + " for log file: " - + ConfigOptions.logFile - ), - MpiConfig, - ) - - try: - logObj.removeHandler(ConfigOptions.logHandle) - except Exception: - err_out_screen_para( - ( - "Unable to remove logging file handle on RANK: " - + str(MpiConfig.rank) - + " for log file: " - + ConfigOptions.logFile ), MpiConfig, ) - try: - ConfigOptions.logHandle.close() - except Exception: - err_out_screen_para( - ( - "Unable to close logging file: " - + ConfigOptions.logFile - + " on RANK: " - + str(MpiConfig.rank) - ), - MpiConfig, - ) - - ConfigOptions.logHandle = None - def check_forcing_bounds(ConfigOptions, input_forcings, MpiConfig): """Check the bounds of forcing variables for reasonable values. diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py index 6948fb98..2d9bd053 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forcingInputMod.py @@ -4,8 +4,6 @@ initializing ESMF grids and regrid objects), etc """ -import logging - import numpy as np from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.config import ( @@ -15,11 +13,12 @@ GeoMetaWrfHydro, ) from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import MpiConfig -from nextgen_forcings_ewts import MODULE_NAME from . import regrid, time_handling, timeInterpMod -LOG = logging.getLogger(MODULE_NAME) +# Use the Error, Warning, and Trapping System Package for logging +import ewts +LOG = ewts.get_logger(ewts.FORCING_ID) class InputForcings: diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py index bc773e42..a0aadf92 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/forecastMod.py @@ -80,7 +80,7 @@ def process_forecasts( # move on. continue - if (not ConfigOptions.ana_flag) or (ConfigOptions.logFile is None): + if not ConfigOptions.ana_flag: if MpiConfig.rank == 0: # If the cycle directory doesn't exist, create it. if not os.path.isdir(fcstCycleOutDir): @@ -93,36 +93,6 @@ def process_forecasts( err_handler.err_out_screen_para(ConfigOptions.errMsg, MpiConfig) err_handler.check_program_status(ConfigOptions, MpiConfig) - # Compose a path to a log file, which will contain information - # about this forecast cycle. - # ConfigOptions.logFile = ConfigOptions.output_dir + "/LOG_" + \ - - if ConfigOptions.ana_flag: - log_time = ConfigOptions.e_date_proc - else: - log_time = ConfigOptions.current_fcst_cycle - - ConfigOptions.logFile = ( - ConfigOptions.scratch_dir - + "/LOG_" - + ConfigOptions.nwmConfig - + ( - "_" - if ConfigOptions.nwmConfig != "long_range" - else "_mem" + str(ConfigOptions.cfsv2EnsMember) + "_" - ) - + ConfigOptions.d_program_init.strftime("%Y%m%d%H%M") - + "_" - + log_time.strftime("%Y%m%d%H%M") - ) - - # Initialize the log file. - try: - err_handler.init_log(ConfigOptions, MpiConfig) - except Exception: - err_handler.err_out_screen_para(ConfigOptions.errMsg, MpiConfig) - err_handler.check_program_status(ConfigOptions, MpiConfig) - # Log information about this forecast cycle if MpiConfig.rank == 0: ConfigOptions.statusMsg = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" @@ -366,13 +336,6 @@ def process_forecasts( err_handler.log_msg(ConfigOptions, MpiConfig) err_handler.check_program_status(ConfigOptions, MpiConfig) - if MpiConfig.rank == 0: - # Close the log file. - try: - err_handler.close_log(ConfigOptions, MpiConfig) - except Exception: - err_handler.err_out_screen_para(ConfigOptions.errMsg, MpiConfig) - # Success.... Now touch an empty complete file for this forecast cycle to indicate # completion in case the code is re-ran. try: diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py index 8902ebbe..794309b2 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/geoMod.py @@ -17,12 +17,9 @@ except ImportError: import ESMF -import logging - -from nextgen_forcings_ewts import MODULE_NAME - -LOG = logging.getLogger(MODULE_NAME) - +# Use the Error, Warning, and Trapping System Package for logging +import ewts +LOG = ewts.get_logger(ewts.FORCING_ID) class GeoMetaWrfHydro: """Abstract class for handling information about the WRF-Hydro domain we are processing forcings too.""" diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/ioMod.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/ioMod.py index 451ee041..cb58095a 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/ioMod.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/ioMod.py @@ -7,7 +7,6 @@ import datetime import gzip -import logging import math import os import shutil @@ -18,11 +17,11 @@ import numpy as np from netCDF4 import Dataset -from nextgen_forcings_ewts import MODULE_NAME - from . import err_handler -LOG = logging.getLogger(MODULE_NAME) +# Use the Error, Warning, and Trapping System Package for logging +import ewts +LOG = ewts.get_logger(ewts.FORCING_ID) if "WGRIB2" not in os.environ: WGRIB2_env = False diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py index 8caa1a93..bd3d1da1 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/regrid.py @@ -25,8 +25,6 @@ except ImportError: import ESMF -import logging - import dask import dask.delayed import netCDF4 as nc @@ -46,7 +44,6 @@ GeoMetaWrfHydro, ) from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import MpiConfig -from nextgen_forcings_ewts import MODULE_NAME from ..esmf_utils import ( esmf_field_retry, @@ -57,7 +54,9 @@ esmf_regridobj_call_retry, ) -LOG = logging.getLogger(MODULE_NAME) +# Use the Error, Warning, and Trapping System Package for logging +import ewts +LOG = ewts.get_logger(ewts.FORCING_ID) if "WGRIB2" not in os.environ: WGRIB2_env = False diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/time_handling.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/time_handling.py index 4ac291b8..3ad2e1dc 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/time_handling.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/core/time_handling.py @@ -2,18 +2,17 @@ # calculations in the forcing engine. import datetime import glob -import logging import math import os import numpy as np import pandas as pd -from nextgen_forcings_ewts import MODULE_NAME - from . import err_handler -LOG = logging.getLogger(MODULE_NAME) +# Use the Error, Warning, and Trapping System Package for logging +import ewts +LOG = ewts.get_logger(ewts.FORCING_ID) NETCDF = "NETCDF" diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py index 2e251cf1..543cc2b0 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py @@ -1,7 +1,6 @@ """Module for processing AORC and NWM data.""" import datetime -import logging import os import re import typing @@ -27,11 +26,12 @@ ConfigOptions, ) from NextGen_Forcings_Engine_BMI.NextGen_Forcings_Engine.core.parallel import MpiConfig -from nextgen_forcings_ewts import MODULE_NAME -zarr.config.set({"async.concurrency": 100}) -LOG = logging.getLogger(MODULE_NAME) +# Use the Error, Warning, and Trapping System Package for logging +import ewts +LOG = ewts.get_logger(ewts.FORCING_ID) +zarr.config.set({"async.concurrency": 100}) class BaseProcessor: """Base class for data processors.""" diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py index a71a2f8c..63b3d666 100755 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/model.py @@ -1,5 +1,4 @@ import datetime -import logging import os from contextlib import contextmanager from time import time @@ -29,9 +28,10 @@ NWMV3ConusProcessor, NWMV3OConusProcessor, ) -from nextgen_forcings_ewts import MODULE_NAME -LOG = logging.getLogger(MODULE_NAME) +# Use the Error, Warning, and Trapping System Package for logging +import ewts +LOG = ewts.get_logger(ewts.FORCING_ID) @contextmanager @@ -283,29 +283,6 @@ def adjust_precip( for force_key in config_options.input_forcings: input_forcing_mod[force_key].skip = False - # Determine log timestamp - if config_options.ana_flag: - log_time = config_options.b_date_proc - else: - log_time = config_options.current_fcst_cycle - - # Compose a path to a log file, which will contain information about this forecast cycle - log_filename = ( - f"LOG_{config_options.nwmConfig}" - f"{'_' if config_options.nwmConfig != 'long_range' else f'_mem{config_options.cfsv2EnsMember}_'}" - f"{config_options.d_program_init.strftime('%Y%m%d%H%M')}_{log_time.strftime('%Y%m%d%H%M')}" - ".log" - ) - config_options.logFile = os.path.join( - config_options.scratch_dir, log_filename - ) - - # Initialize logging - try: - err_handler.init_log(config_options, mpi_config) - except Exception: - err_handler.err_out_screen_para(config_options.errMsg, mpi_config) - err_handler.check_program_status(config_options, mpi_config) return ( config_options, diff --git a/nextgen_forcings_ewts/pyproject.toml b/nextgen_forcings_ewts/pyproject.toml deleted file mode 100644 index 01dafb11..00000000 --- a/nextgen_forcings_ewts/pyproject.toml +++ /dev/null @@ -1,8 +0,0 @@ -[build-system] -requires = ["setuptools>=61"] -build-backend = "setuptools.build_meta" - -[project] -name = "nextgen-forcings-ewts" -version = "0.1.0" -requires-python = ">=3.6" diff --git a/nextgen_forcings_ewts/src/nextgen_forcings_ewts/__init__.py b/nextgen_forcings_ewts/src/nextgen_forcings_ewts/__init__.py deleted file mode 100644 index fd849a3c..00000000 --- a/nextgen_forcings_ewts/src/nextgen_forcings_ewts/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -""" -Error Warning and Trapping System (EWTS) Package API - -This package provides a centralized, named logging configuration for the -Error, Warning, and Trapping System used throughout the codebase. - -EWTS configures a single, shared logger in the Python logging framework, -identified by a fixed module name. All modules that participate in EWTS -logging retrieve this logger by name via the standard logging API. - -Logging configuration should be performed once at application startup by -calling configure_logging(). The configuration function is idempotent: -subsequent calls have no effect and will not reconfigure handlers or levels. - -The logger name is exposed to allow any module to obtain the configured -logger without importing internal implementation details. - -Typical usage: - - At application startup: - from nextgen_forcings_ewts import configure_logging - configure_logging() - - Within other modules: - import logging - from nextgen_forcings_ewts import MODULE_NAME - - LOG = logging.getLogger(MODULE_NAME) -""" - -from .constants import MODULE_NAME -from .config import configure_logging - -__all__ = ["MODULE_NAME", "configure_logging"] diff --git a/nextgen_forcings_ewts/src/nextgen_forcings_ewts/config.py b/nextgen_forcings_ewts/src/nextgen_forcings_ewts/config.py deleted file mode 100644 index 81d0274e..00000000 --- a/nextgen_forcings_ewts/src/nextgen_forcings_ewts/config.py +++ /dev/null @@ -1,127 +0,0 @@ -""" -Logging configuration for the Error Warning and Trapping System (EWTS). - -This module defines the centralized logging configuration used by EWTS. -It is responsible for creating and configuring a single, named logger -within the Python logging framework, based on environment variables -provided by the runtime environment (e.g., ngen). - -Logging configuration is performed via configure_logging(), which applies -handlers, formatters, and log levels to the EWTS logger. The configuration -function is idempotent: once the logger has been initialized, subsequent -calls return immediately without modifying the existing configuration. - -Configuration behavior is controlled by environment variables, whose names -are defined in constants.py: - - - EV_EWTS_LOGGING: - Enables or disables EWTS logging. If set to "DISABLED", logging is - disabled entirely for the EWTS logger. If unset, logging is enabled - by default. - - - EV_MODULE_LOGLEVEL: - Specifies the log level for the EWTS logger. Supported values include - standard Python logging levels as well as ngen-style levels (e.g., - "SEVERE", "FATAL"), which are translated to Python equivalents. - -Log output is directed to a file determined by the path-resolution utilities -in paths.py. If a log file cannot be created, logging falls back to stdout. - -This module does not expose logging APIs directly; callers are expected to -retrieve the configured logger by name using logging.getLogger(MODULE_NAME). -""" - -import logging -import sys -import os - -from .constants import ( - MODULE_NAME, - EV_EWTS_LOGGING, - EV_MODULE_LOGLEVEL, - LOG_MODULE_NAME_LEN, -) -from .formatter import CustomFormatter -from .paths import get_log_file_path -from .helper import getenv_any - -def translate_ngwpc_log_level(level: str) -> str: - level = level.strip().upper() - return { - "SEVERE": "ERROR", - "FATAL": "CRITICAL", - }.get(level, level) - - -def force_info(handler, logger, msg, *args): - record = logger.makeRecord( - logger.name, - logging.INFO, - __file__, - 0, - msg, - args, - None, - ) - handler.emit(record) - - -def configure_logging(): - ''' - Set logging level and specify logger configuration based on environment variables set by ngen - ''' - logger = logging.getLogger(MODULE_NAME) - - if getattr(logger, "_initialized", False): - return logger # logger already initialized, nothing else to do - - # Default to enabled if flag not set or is set to disabled - raw_value = getenv_any(EV_EWTS_LOGGING, "") - normalized = (raw_value or "").strip().lower() # convert None or "" to "", lowercase for easy comparison - - # Determine if logging is enabled - enabled = normalized != "disabled" - - # Inform user if logging is enabled by default (env not explicitly set to "enabled") - if enabled and normalized not in ("enabled",): - print(f"{EV_EWTS_LOGGING} not explicitly set to 'ENABLED'; logging ENABLED by default", flush=True) - - if not enabled: - logger.disabled = True - logger._initialized = True - print(f"Module {MODULE_NAME} Logging DISABLED", flush=True) - return logger - - print(f"Module {MODULE_NAME} Logging ENABLED", flush=True) - - logFilePath, appendEntries = get_log_file_path() - - handler = ( - logging.FileHandler(logFilePath, mode="a" if appendEntries else "w") - if logFilePath - else logging.StreamHandler(sys.stdout) - ) - - log_level = translate_ngwpc_log_level( - getenv_any(EV_MODULE_LOGLEVEL, "INFO") - ) - - module_fmt = MODULE_NAME.upper().ljust(LOG_MODULE_NAME_LEN)[:LOG_MODULE_NAME_LEN] - - formatter = CustomFormatter( - fmt=f"%(asctime)s.%(msecs)03d {module_fmt} %(levelname_padded)s %(message)s", - datefmt="%Y-%m-%dT%H:%M:%S", - ) - handler.setFormatter(formatter) - - # Setup logger - logger.handlers.clear() # Clear any default handlers - logger.setLevel(log_level) - logger.addHandler(handler) - - # Write log level INFO message to log regradless of the actual log level - force_info(handler, logger, "Log level set to %s", log_level) - print(f"Module {MODULE_NAME} Log Level set to {log_level}", flush=True) - - logger._initialized = True - return logger diff --git a/nextgen_forcings_ewts/src/nextgen_forcings_ewts/constants.py b/nextgen_forcings_ewts/src/nextgen_forcings_ewts/constants.py deleted file mode 100644 index 7ca26727..00000000 --- a/nextgen_forcings_ewts/src/nextgen_forcings_ewts/constants.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Constants and configuration keys for the Error Warning and Trapping System (EWTS). - -This module defines all constant values used by EWTS for logging configuration, -environment variable integration, and log file naming. These values represent -the stable interface between EWTS, ngen, and participating Python modules. - -Constants are grouped into two categories: - - 1) Module-specific constants: - Values that uniquely identify the current ngen module, including the - logger name and module-specific environment variables. - - 2) Common constants: - Values shared across ngen modules that control global logging behavior, - filesystem layout, and integration with the ngen runtime environment. - -These constants are intentionally centralized to ensure consistent behavior -across the codebase and to avoid hard-coded strings in implementation logic. -Callers should treat these values as read-only. -""" - - -# Values unique to each ngen module -MODULE_NAME = "Forcing" -EV_MODULE_LOGLEVEL = "FORCING_LOGLEVEL" # This modules log level -EV_MODULE_LOGFILEPATH = "FORCING_LOGFILEPATH" # This modules log full log filename - -# Values common to all ngen modules -EV_NGEN_LOGFILEPATH = "NGEN_LOG_FILE_PATH" # Environment variable name with the log file location typically set by ngen -EV_EWTS_LOGGING = "NGEN_EWTS_LOGGING" # Environment variable name with the enable/disable state for the Error Warning - # and Trapping System typically set by ngen - -DS = "/" # Directory separator -LOG_DIR_DEFAULT = "run-logs" # Default parent log directory string if env var empty & ngencerf doesn't exist -LOG_DIR_NGENCERF = "/ngencerf/data" # ngenCERF log directory string if environement var empty. -LOG_FILE_EXT = "log" # Log file name extension -LOG_MODULE_NAME_LEN = 8 # Width of module name for log entries - - diff --git a/nextgen_forcings_ewts/src/nextgen_forcings_ewts/formatter.py b/nextgen_forcings_ewts/src/nextgen_forcings_ewts/formatter.py deleted file mode 100644 index a28bc75b..00000000 --- a/nextgen_forcings_ewts/src/nextgen_forcings_ewts/formatter.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -Custom log record formatting for the Error Warning and Trapping System (EWTS). - -This module defines a custom logging formatter used by EWTS to produce -consistent, ngen-compatible log output across all participating modules. - -The formatter applies the following behaviors: - - - Forces all timestamps to UTC, independent of system locale settings. - - Formats timestamps with millisecond precision. - - Maps Python logging levels to ngen-style severity names - (e.g., ERROR → SEVERE, CRITICAL → FATAL). - - Pads and normalizes level names to fixed width for column alignment. - - Strips trailing whitespace and newline characters from log messages. - -The formatter operates entirely within the Python logging framework and does -not modify logger configuration or handler behavior. It is intended to be used -by the EWTS logging configuration layer and not instantiated directly by -application code. -""" - -import logging -import time - -# Define PERFORMANCE logging level -PERFORMANCE_LEVEL_NUM = 15 # Between DEBUG (10) and INFO (20) -logging.PERFORMANCE = PERFORMANCE_LEVEL_NUM # <-- add as logging constant -logging.addLevelName(PERFORMANCE_LEVEL_NUM, "PERFORMANCE") - -# Optional: add a convenience method to logger objects -def performance(self, message, *args, **kwargs): - if self.isEnabledFor(logging.PERFORMANCE): - self._log(logging.PERFORMANCE, message, args, **kwargs) - -logging.Logger.performance = performance - -class CustomFormatter(logging.Formatter): - LEVEL_NAME_MAP = { - logging.DEBUG: "DEBUG", - logging.INFO: "INFO", - PERFORMANCE_LEVEL_NUM: "PERFORM", - logging.WARNING: "WARNING", - logging.ERROR: "SEVERE", - logging.CRITICAL: "FATAL" - } - - # Apply custom formatter (UTC timestamps applied only to this formatter) - def converter(self, timestamp): - """Override time converter to return UTC time tuple""" - return time.gmtime(timestamp) - - def formatTime(self, record, datefmt=None): - """Use our UTC converter""" - ct = self.converter(record.created) - if datefmt: - return time.strftime(datefmt, ct) - t = time.strftime("%Y-%m-%d %H:%M:%S", ct) - return f"{t},{int(record.msecs):03d}" - - def format(self, record): - # Strip trailing whitespace/newlines from the message - if record.msg: - record.msg = str(record.msg).rstrip() - - # Map level names - original_levelname = record.levelname - record.levelname = self.LEVEL_NAME_MAP.get(record.levelno, original_levelname) - record.levelname_padded = record.levelname.ljust(7)[:7] # Exactly 7 chars - formatted = super().format(record) - - # Restore original levelname - record.levelname = original_levelname # Restore original in case it's reused - return formatted diff --git a/nextgen_forcings_ewts/src/nextgen_forcings_ewts/helper.py b/nextgen_forcings_ewts/src/nextgen_forcings_ewts/helper.py deleted file mode 100644 index db90ea33..00000000 --- a/nextgen_forcings_ewts/src/nextgen_forcings_ewts/helper.py +++ /dev/null @@ -1,32 +0,0 @@ -import os - -# NOTE: -# ngen sets some env vars from C++ after the Python interpreter has started. -# In embedded Python, os.environ may not reflect those changes. -# getenv_any() falls back to libc getenv() and syncs os.environ. -def getenv_any(key: str, default: str = "") -> str: - """ - Get an environment variable reliably even when it is set from C/C++ - after the Python interpreter has started (embedded Python). - Prefers os.environ/os.getenv, falls back to libc getenv. - """ - # First try Python's mapping - v = os.environ.get(key) - if v is not None: - return v - - # Fallback: direct libc getenv (sees process env even if Python mapping is stale) - try: - import ctypes, ctypes.util - libc = ctypes.CDLL(ctypes.util.find_library("c")) - libc.getenv.restype = ctypes.c_char_p - b = libc.getenv(key.encode("utf-8")) - if not b: - return default - s = b.decode("utf-8") - - # Sync back into os.environ so future lookups work normally - os.environ[key] = s - return s - except Exception: - return default diff --git a/nextgen_forcings_ewts/src/nextgen_forcings_ewts/paths.py b/nextgen_forcings_ewts/src/nextgen_forcings_ewts/paths.py deleted file mode 100644 index 871233df..00000000 --- a/nextgen_forcings_ewts/src/nextgen_forcings_ewts/paths.py +++ /dev/null @@ -1,117 +0,0 @@ -""" -Log file path resolution utilities for the Error Warning and Trapping System (EWTS). - -This module provides helper functions for constructing and validating log file -paths used by the EWTS logging configuration. Log file selection follows a -well-defined precedence based on environment variables and runtime availability. - -Log file path precedence: - - 1. If the NGEN-provided log file path is available via the environment variable - defined in EV_NGEN_LOGFILEPATH, use that path. - - 2. Otherwise, create a default, module-specific log file: - 2.1) Create a base log directory under the ngenCERF data directory if it - exists; otherwise fall back to the user's home directory. - 2.2) Create a child directory using the current username if available, - otherwise use the current UTC date (YYYYMMDD). - 2.3) Construct a log filename using the module name and a UTC timestamp. - -The resolved log file path is validated by attempting to open the file. Upon -successful creation or reuse, the full log file path is stored in the -EV_MODULE_LOGFILEPATH environment variable so subsequent calls reuse the same -file. If log file creation fails, entries will be written to stdout. - -This module does not configure loggers directly; it only resolves filesystem -paths and associated metadata required by the logging configuration layer. -""" - -import getpass -import os -from datetime import datetime, timezone - -from .constants import ( - MODULE_NAME, - EV_NGEN_LOGFILEPATH, - EV_MODULE_LOGFILEPATH, - DS, - LOG_DIR_DEFAULT, - LOG_DIR_NGENCERF, - LOG_FILE_EXT, -) -from .helper import getenv_any - - -def create_timestamp(date_only=False, iso=False, append_ms=False): - now = datetime.now(timezone.utc) - - if date_only: - ts = now.strftime("%Y%m%d") - elif iso: - ts = now.strftime("%Y-%m-%dT%H:%M:%S") - else: - ts = now.strftime("%Y%m%dT%H%M%S") - - if append_ms: - ts += f".{now.microsecond // 1000:03d}" - - return ts - -def get_log_file_path(): - # Determine the log file path using the following precedence: - # 1) Use the ngen-provided log file path if available in the NGEN_LOG_FILE_PATH environment variable - # 2) Otherwise, create a default module-specific log file using the module name and a UTC timestamp. - # 2.1) First create a subdirectory under the ngenCERF data directory if available, otherwise the user home directory. - # 2.2) Next create a subdirectory name using the username, if available, otherwise use the YYYYMMDD. - # 2.3) Attempt to open the log file and upon failure, use stdout. - - appendEntries = True - moduleLogFileExists = False - - # Determine if a log file has laready been opened for this module (either the ngen log or default) - moduleEnvVar = getenv_any(EV_MODULE_LOGFILEPATH, "").strip() - if moduleEnvVar: - logFilePath = moduleEnvVar - moduleLogFileExists = True - else: - ngenEnvVar = getenv_any(EV_NGEN_LOGFILEPATH, "").strip() - if ngenEnvVar: - logFilePath = ngenEnvVar - else: - print(f"Module {MODULE_NAME} Env var {EV_NGEN_LOGFILEPATH} not found. Creating default log name.") - appendEntries = False - baseDir = ( - f"{LOG_DIR_NGENCERF}{DS}{LOG_DIR_DEFAULT}" - if os.path.isdir(LOG_DIR_NGENCERF) - else f"{os.path.expanduser('~')}{DS}{LOG_DIR_DEFAULT}" - ) - try: - os.makedirs(baseDir, exist_ok=True) - - childDir = getpass.getuser() or create_timestamp(True) - logFileDir = f"{baseDir}{DS}{childDir}" - os.makedirs(logFileDir, exist_ok=True) - - logFilePath = ( - f"{logFileDir}{DS}{MODULE_NAME}_{create_timestamp()}.{LOG_FILE_EXT}" - ) - except Exception as e: - print(f"Module {MODULE_NAME} {e}", flush=True) - logFilePath = "" - - # Ensure log file can be opened and set module env var - try: - if (logFilePath): - mode = "a" if appendEntries else "w" - with open(logFilePath, mode): - pass - if not moduleLogFileExists: - os.environ[EV_MODULE_LOGFILEPATH] = logFilePath - print(f"Module {MODULE_NAME} Log File: {logFilePath}", flush=True) - else: - raise IOError - except Exception: - print(f"Module {MODULE_NAME} Unable to open log file: {logFilePath}", flush=True) - print(f"Module {MODULE_NAME} Log entries will be writen to stdout", flush=True) - - return logFilePath, appendEntries diff --git a/pyproject.toml b/pyproject.toml index 8759a65f..2cb6ae2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,8 +53,7 @@ build-backend = "setuptools.build_meta" where = ["NextGen_Forcings_Engine_BMI", "nextgen_forcings_ewts/src", "."] include = ["NextGen_Forcings_Engine*", "Forcing_Extraction_Scripts*", - "ESMF_Mesh_Domain_Configuration_Production*", - "nextgen_forcings_ewts*"] + "ESMF_Mesh_Domain_Configuration_Production*"] namespaces = false [tool.setuptools.dynamic] diff --git a/tests/nextgen_forcings_ewts/conftest.py b/tests/nextgen_forcings_ewts/conftest.py deleted file mode 100644 index 2b6b9c90..00000000 --- a/tests/nextgen_forcings_ewts/conftest.py +++ /dev/null @@ -1,25 +0,0 @@ -import logging -import pytest - - -@pytest.fixture -def clean_ewts_env(monkeypatch): - """ - Ensure EWTS-related environment variables are unset and - logging is reset before each test. - """ - # EWTS / module env vars - monkeypatch.delenv("NGEN_LOG_FILE_PATH", raising=False) - monkeypatch.delenv("FORCING_LOGLEVEL", raising=False) - monkeypatch.delenv("FORCING_LOGFILEPATH", raising=False) - monkeypatch.delenv("NGEN_EWTS_LOGGING", raising=False) - - # Reset logging state (important!) - logging.shutdown() - for handler in logging.root.handlers[:]: - logging.root.removeHandler(handler) - - yield - - # Cleanup after test (defensive) - logging.shutdown() diff --git a/tests/nextgen_forcings_ewts/test_config.py b/tests/nextgen_forcings_ewts/test_config.py deleted file mode 100644 index 8d5993b7..00000000 --- a/tests/nextgen_forcings_ewts/test_config.py +++ /dev/null @@ -1,81 +0,0 @@ -import pytest - -import logging -from nextgen_forcings_ewts.config import configure_logging, translate_ngwpc_log_level -from nextgen_forcings_ewts.constants import MODULE_NAME, EV_EWTS_LOGGING - -# ------------------------------ -def test_configure_logging_default(clean_ewts_env): - logger = configure_logging() - - assert logger.name == MODULE_NAME - assert logger.level == logging.INFO - assert not logger.disabled - -# ------------------------------ -def test_configure_logging_idempotent(clean_ewts_env): - logger1 = configure_logging() - logger2 = configure_logging() - - assert logger1 is logger2 - assert getattr(logger1, "_initialized", False) - -# ------------------------------ -@pytest.mark.parametrize("inp,expected", [ - ("INFO", "INFO"), - ("SeVeRe", "ERROR"), - ("fatal", "CRITICAL"), - (" debug ", "DEBUG"), -]) -def test_translate_ngwpc_log_level(inp, expected): - assert translate_ngwpc_log_level(inp) == expected - -# ------------------------------ -@pytest.mark.parametrize("env_value,expected_enabled", [ - (None, True), # default: enabled - ("DISABLED", False), - ("ENABLED", True), - ("disabled", False), - ("enabled", True), - ("anystring", True), - ("", True), -]) -@pytest.mark.parametrize("level_input,expected_level", [ - ("DEBUG", logging.DEBUG), - ("INFO", logging.INFO), - ("SEVERE", logging.ERROR), - ("FATAL", logging.CRITICAL), -]) -def test_ewts_logger_matrix(clean_ewts_env, monkeypatch, capsys, env_value, expected_enabled, level_input, expected_level): - # Set environment variables - if env_value is None: - monkeypatch.delenv("NGEN_EWTS_LOGGING", raising=False) - else: - monkeypatch.setenv("NGEN_EWTS_LOGGING", env_value) - - monkeypatch.setenv("FORCING_LOGLEVEL", level_input) - - # Force logger re-initialization - logger = logging.getLogger(MODULE_NAME) - logger.handlers.clear() - logger._initialized = False - logger.disabled = False # ensure proper reset - - # Configure logger - logger = configure_logging() - - # Capture stdout - captured = capsys.readouterr() - - # Assertions - assert logger.name == MODULE_NAME - assert (not logger.disabled) == expected_enabled # True if enabled - if expected_enabled: - assert logger.level == expected_level - - # Assertions for default-enabled print - if expected_enabled and (env_value is None or env_value not in ("ENABLED", "enabled")): - assert f"{EV_EWTS_LOGGING} not explicitly set" in captured.out - else: - assert f"{EV_EWTS_LOGGING} not explicitly set" not in captured.out - diff --git a/tests/nextgen_forcings_ewts/test_constants.py b/tests/nextgen_forcings_ewts/test_constants.py deleted file mode 100644 index 79d4d6de..00000000 --- a/tests/nextgen_forcings_ewts/test_constants.py +++ /dev/null @@ -1,10 +0,0 @@ -from nextgen_forcings_ewts.constants import ( - MODULE_NAME, - LOG_MODULE_NAME_LEN, -) - -def test_module_name_is_string(): - assert isinstance(MODULE_NAME, str) - -def test_module_name_length_fits_field(): - assert len(MODULE_NAME) <= LOG_MODULE_NAME_LEN diff --git a/tests/nextgen_forcings_ewts/test_formatter.py b/tests/nextgen_forcings_ewts/test_formatter.py deleted file mode 100644 index 57cabb4b..00000000 --- a/tests/nextgen_forcings_ewts/test_formatter.py +++ /dev/null @@ -1,66 +0,0 @@ -import logging -import pytest -from nextgen_forcings_ewts.formatter import CustomFormatter -from nextgen_forcings_ewts.constants import MODULE_NAME - -@pytest.fixture -def formatter(): - fmt = "%(asctime)s %(levelname_padded)s %(message)s" - return CustomFormatter(fmt=fmt, datefmt="%Y-%m-%dT%H:%M:%S") - -@pytest.mark.parametrize( - "level,expected", - [ - (logging.DEBUG, "DEBUG"), - (logging.PERFORMANCE, "PERFORM"), - (logging.INFO, "INFO"), - (logging.WARNING, "WARNING"), - (logging.ERROR, "SEVERE"), - (logging.CRITICAL, "FATAL"), - ] -) -def test_level_name_mapping(formatter, level, expected): - record = logging.LogRecord( - name=MODULE_NAME, - level=level, - pathname="test", - lineno=0, - msg="Test message", - args=None, - exc_info=None - ) - formatted = formatter.format(record) - # Level name should appear in formatted string - assert expected in formatted - -def test_utc_timestamp(formatter): - record = logging.LogRecord( - name=MODULE_NAME, - level=logging.INFO, - pathname="test", - lineno=0, - msg="UTC test", - args=None, - exc_info=None - ) - formatted = formatter.format(record) - # Timestamp should be in UTC format "YYYY-MM-DDTHH:MM:SS" - ts_str = formatted.split()[0] - from datetime import datetime - dt = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S") - # It's enough to check it parses without error - -def test_trailing_whitespace_stripped(formatter): - record = logging.LogRecord( - name=MODULE_NAME, - level=logging.INFO, - pathname="test", - lineno=0, - msg="Message with space \n", - args=None, - exc_info=None - ) - formatted = formatter.format(record) - # Trailing whitespace/newline should be removed - assert " \n" not in formatted - assert formatted.endswith("Message with space") diff --git a/tests/nextgen_forcings_ewts/test_paths.py b/tests/nextgen_forcings_ewts/test_paths.py deleted file mode 100644 index a7011a85..00000000 --- a/tests/nextgen_forcings_ewts/test_paths.py +++ /dev/null @@ -1,115 +0,0 @@ -import os -import getpass -from datetime import datetime -import pytest -from nextgen_forcings_ewts import paths -from nextgen_forcings_ewts.paths import create_timestamp, get_log_file_path -from nextgen_forcings_ewts.constants import MODULE_NAME, EV_MODULE_LOGFILEPATH, EV_NGEN_LOGFILEPATH - -# ------------------------------- -# Fixture for a clean log environment -# ------------------------------- -@pytest.fixture -def clean_log_env(tmp_path, monkeypatch): - """Set up a temporary log environment and clean env vars. - - Yields a dict with: - tmp_dir : Path of temporary base directory - monkeypatch : the pytest monkeypatch object for further tweaks - """ - # Clear env vars - monkeypatch.delenv(EV_MODULE_LOGFILEPATH, raising=False) - monkeypatch.delenv(EV_NGEN_LOGFILEPATH, raising=False) - - # Patch constants to use tmp_path - monkeypatch.setattr(paths, "LOG_DIR_NGENCERF", tmp_path) - monkeypatch.setattr(paths, "LOG_DIR_DEFAULT", "run-logs") - - yield {"tmp_dir": tmp_path, "monkeypatch": monkeypatch} - - -# ------------------------------- -# Tests for create_timestamp() -# ------------------------------- -def test_create_timestamp_default(): - ts = create_timestamp() - assert len(ts) >= 15 - assert "T" in ts - -def test_create_timestamp_date_only(): - ts = create_timestamp(date_only=True) - assert len(ts) == 8 - -def test_create_timestamp_iso(): - ts = create_timestamp(iso=True) - assert "T" in ts and "-" in ts and ":" in ts - -def test_create_timestamp_append_ms(): - ts = create_timestamp(append_ms=True) - assert "." in ts - - -# ------------------------------- -# Tests for get_log_file_path() -# ------------------------------- -def test_get_log_file_path_uses_module_env(clean_log_env): - tmp_path = clean_log_env["tmp_dir"] - monkeypatch = clean_log_env["monkeypatch"] - - logfile = tmp_path / "test_module.log" - monkeypatch.setenv(EV_MODULE_LOGFILEPATH, str(logfile)) - - path, append = get_log_file_path() - assert path == str(logfile) - assert append is True - - -def test_get_log_file_path_uses_ngen_env(clean_log_env): - monkeypatch = clean_log_env["monkeypatch"] - tmp_path = clean_log_env["tmp_dir"] - - monkeypatch.delenv(EV_MODULE_LOGFILEPATH, raising=False) - ngen_file = tmp_path / "ngen.log" - monkeypatch.setenv(EV_NGEN_LOGFILEPATH, str(ngen_file)) - - path, append = get_log_file_path() - assert path == str(ngen_file) - assert append is True - - -def test_get_log_file_path_creates_user_subdir(clean_log_env): - tmp_path = clean_log_env["tmp_dir"] - monkeypatch = clean_log_env["monkeypatch"] - - monkeypatch.delenv(EV_MODULE_LOGFILEPATH, raising=False) - monkeypatch.delenv(EV_NGEN_LOGFILEPATH, raising=False) - - # Use real username - monkeypatch.setattr(getpass, "getuser", lambda: "alice") - - path, append = get_log_file_path() - - # Subdirectory should be username - subdir = os.path.basename(os.path.dirname(path)) - assert subdir == "alice" - assert path.endswith(".log") - assert os.path.exists(path) - - -def test_get_log_file_path_fallback_username(clean_log_env): - tmp_path = clean_log_env["tmp_dir"] - monkeypatch = clean_log_env["monkeypatch"] - - monkeypatch.delenv(EV_MODULE_LOGFILEPATH, raising=False) - monkeypatch.delenv(EV_NGEN_LOGFILEPATH, raising=False) - - # Simulate getuser() returning None - monkeypatch.setattr(getpass, "getuser", lambda: None) - - path, append = get_log_file_path() - - subdir = os.path.basename(os.path.dirname(path)) - # Should fall back to YYYYMMDD - assert len(subdir) == 8 and subdir.isdigit() - assert path.endswith(".log") - assert os.path.exists(path) From aedb3827436873cf7f73f1be21c3142983b76a35 Mon Sep 17 00:00:00 2001 From: Miguel Pena Date: Mon, 16 Mar 2026 19:41:11 -0700 Subject: [PATCH 2/3] added ewts to Dockerfile.bmi-forcings and updated cicd file with ewts build arguments --- .github/workflows/cicd.yml | 10 +++++++ Dockerfile.bmi-forcings | 55 +++++++++++++++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 7a188b42..e32b1b19 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -41,6 +41,14 @@ on: description: 'SFINCS_BASE_TAG' required: false type: string + EWTS_ORG: + description: 'EWTS_ORG' + required: false + type: string + EWTS_REF: + description: 'EWTS_REF' + required: false + type: string permissions: contents: read @@ -262,6 +270,8 @@ jobs: BMI_BASE_TAG=${{ needs.setup.outputs.bmi_base_tag }} BMI_BASE_DIGEST=${{ needs.setup.outputs.bmi_base_digest }} BMI_BASE_REVISION=${{ needs.setup.outputs.bmi_base_revision }} + EWTS_ORG=${{ inputs.EWTS_ORG || 'NGWPC' }} + EWTS_REF=${{ inputs.EWTS_REF || 'development' }} IMAGE_SOURCE=https://github.com/${{ github.repository }} IMAGE_VENDOR=${{ github.repository_owner }} IMAGE_VERSION=${{ needs.setup.outputs.clean_ref }} diff --git a/Dockerfile.bmi-forcings b/Dockerfile.bmi-forcings index 541dda24..a828e557 100644 --- a/Dockerfile.bmi-forcings +++ b/Dockerfile.bmi-forcings @@ -172,6 +172,53 @@ ENV OMPI_MCA_btl_base_warn_component_unused=0 # Reset SHELL so we're not locked into the gcc-10 build environment SHELL ["/bin/bash", "-c"] +# ── EWTS (Error, Warning and Trapping System) +# ngen-bmi-forcing only needs the EWTS Python package (not the C/C++/Fortran +# libraries or ngen bridge that the full ngen image requires). +# +# Build args – override at build time to pin a branch, tag, or full commit SHA: +# docker build --build-arg EWTS_REF=v1.2.3 ... +# docker build --build-arg EWTS_REF=abc123def456 ... +ARG EWTS_ORG=NGWPC +ARG EWTS_REF=development + +# Clone nwm-ewts, install the Python package, capture git metadata for +# provenance, then remove the source tree. +# Try shallow clone by branch/tag name first; fall back to full clone + checkout +# for bare commit SHAs (which git clone -b doesn't support). +# +# NOTE: Unlike the ngen Dockerfile, clone + pip install + cleanup are kept in a +# single RUN so the source tree never persists in a layer. In ngen the split is +# safe because cmake installs the wheel to /opt/ewts before the source is removed; +# here there is no cmake step, so the source must remain until pip finishes. +RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \ + set -eux && \ + (git clone --depth 1 -b "${EWTS_REF}" \ + "https://github.com/${EWTS_ORG}/nwm-ewts.git" /tmp/nwm-ewts \ + || (git clone "https://github.com/${EWTS_ORG}/nwm-ewts.git" /tmp/nwm-ewts && \ + cd /tmp/nwm-ewts && git checkout "${EWTS_REF}")) && \ + cd /tmp/nwm-ewts && \ + # ── Capture EWTS git provenance ── + # Saved as a JSON sidecar so the git-info step below can merge EWTS + # metadata alongside ngen-bmi-forcing. + jq -n \ + --arg commit_hash "$(git rev-parse HEAD)" \ + --arg branch "$(git branch -r --contains HEAD 2>/dev/null | grep -v '\->' | sed 's|origin/||' | head -n1 | xargs || echo "${EWTS_REF}")" \ + --arg tags "$(git tag --points-at HEAD 2>/dev/null | tr '\n' ' ')" \ + --arg author "$(git log -1 --pretty=format:'%an')" \ + --arg commit_date "$(date -u -d @$(git log -1 --pretty=format:'%ct') +'%Y-%m-%d %H:%M:%S UTC')" \ + --arg message "$(git log -1 --pretty=format:'%s' | tr '\n' ';')" \ + --arg build_date "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" \ + '{"nwm-ewts": {commit_hash: $commit_hash, branch: $branch, tags: $tags, author: $author, commit_date: $commit_date, message: $message, build_date: $build_date}}' \ + > /ngen-app/nwm-ewts_git_info.json && \ + # ── Install the EWTS Python package into the venv ── + # This is what makes "import ewts" work for ngen-bmi-forcing. + # For example, bmi_model.py does: import ewts; LOG = ewts.get_logger(ewts.FORCING_ID) + pip install /tmp/nwm-ewts/runtime/python/ewts && \ + # ── Cleanup ── + cd / && \ + rm -rf /tmp/nwm-ewts + COPY . /ngen-app/ngen-forcing/ # Install ngen-forcing as a Python package @@ -201,7 +248,13 @@ RUN set -eux; \ --arg message "$(git log -1 --pretty=format:'%s' | tr '\n' ';')" \ --arg build_date "$(date -u +'%Y-%m-%d %H:%M:%S UTC')" \ "{\"ngen-bmi-forcing\": {commit_hash: \$commit_hash, branch: \$branch, tags: \$tags, author: \$author, commit_date: \$commit_date, message: \$message, build_date: \$build_date}}" \ - > $GIT_INFO_PATH + > $GIT_INFO_PATH; \ + # Merge EWTS git provenance if it exists + if [ -f /ngen-app/nwm-ewts_git_info.json ]; then \ + jq -s 'add' $GIT_INFO_PATH /ngen-app/nwm-ewts_git_info.json > ${GIT_INFO_PATH}.tmp; \ + mv ${GIT_INFO_PATH}.tmp $GIT_INFO_PATH; \ + rm -f /ngen-app/nwm-ewts_git_info.json; \ + fi WORKDIR / From d1a4e55628d16f708242f4d9f6b13b88a1b75437 Mon Sep 17 00:00:00 2001 From: "kyle.larkin" Date: Tue, 17 Mar 2026 12:32:45 -0400 Subject: [PATCH 3/3] removes LOG from parameter in definition --- .../NextGen_Forcings_Engine/historical_forcing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py index 543cc2b0..7fdbd06b 100644 --- a/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py +++ b/NextGen_Forcings_Engine_BMI/NextGen_Forcings_Engine/historical_forcing.py @@ -83,7 +83,7 @@ def reprojected_ymax(self) -> float: return self.bounds[3] @contextmanager - def timing_block(self, step_str: str, log_callable: typing.Callable = LOG.debug): + def timing_block(self, step_str: str, log_callable: typing.Callable = None): """Context manager for timing code execution. Args: @@ -91,6 +91,8 @@ def timing_block(self, step_str: str, log_callable: typing.Callable = LOG.debug) log_callable: Callable used for sending the log message. Defaults to LOG.debug. """ + if log_callable is None: + log_callable = LOG.debug start = perf_counter() log_callable(f" Starting {step_str}") yield