From 62762a18fff524c176814b85a13aa1a918df7fc0 Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Fri, 1 Aug 2025 11:40:49 -0500 Subject: [PATCH 01/17] allow push From d96d464e79c7510f9cb0fcfa59d9d4ca80246ffc Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Tue, 5 Aug 2025 16:02:44 -0500 Subject: [PATCH 02/17] address copilot comments --- python_utils/aqm-data-sync/README.md | 6 ++++++ python_utils/aqm-data-sync/src/aqm_data_sync/core.py | 5 +++-- .../src/aqm_data_sync/logging_aqm_data_sync.py | 6 ++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/python_utils/aqm-data-sync/README.md b/python_utils/aqm-data-sync/README.md index b19eedc..c1c345f 100644 --- a/python_utils/aqm-data-sync/README.md +++ b/python_utils/aqm-data-sync/README.md @@ -11,6 +11,12 @@ conda env create -f environment.yml conda run -n aqm-data-sync pip install . ``` +# Testing + +```shell +pytest test +``` + # Usage ```shell diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py index 7f8bcb7..213eb19 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py @@ -34,7 +34,7 @@ def system_max_concurrent_requests(self) -> int | None: ("aws", "configure", "get", "default.s3.max_concurrent_requests") ) except subprocess.CalledProcessError: - LOGGER("could not retrieve max_concurrent_requests", level=logging.WARN) + LOGGER("could not retrieve max_concurrent_requests", level=logging.WARNING) return None else: return int(raw_output) @@ -84,6 +84,7 @@ class UseCaseAeromma(UseCase): @classmethod def _initialize_model_(cls, values: dict) -> dict: for key in ("first_cycle_date", "last_cycle_date"): + # Allow values to be initialized by the parent model with empty strings if values.get(key, "") is None: values.pop(key) return values @@ -142,7 +143,7 @@ def _update_include_templates_(self, cmd: list[str]) -> None: while True: LOGGER(f"{ctr=}, {curr_cycle_date=}") if ctr > 1000: - LOGGER("", exc_info=ValueError(f"{ctr=} - don't be ridiculous")) + LOGGER("", exc_info=ValueError(f"{ctr=} - Exceeded max iterations")) include_templates = self._create_include_templates_for_cycle_date_(curr_cycle_date) if ctr == 0: LOGGER("adding restart file download") diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/logging_aqm_data_sync.py b/python_utils/aqm-data-sync/src/aqm_data_sync/logging_aqm_data_sync.py index cd02f90..74a6398 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/logging_aqm_data_sync.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/logging_aqm_data_sync.py @@ -21,7 +21,7 @@ def __call__( self, msg, level=logging.INFO, - exc_info: Exception = None, + exc_info: Exception | None = None, stacklevel: int = 2, ): """ @@ -74,7 +74,9 @@ def initialize( "loggers": { _PROJECT_NAME: { "handlers": ["default"], - "level": getattr(logging, log_level.value.upper()), # pylint: disable=no-member + "level": getattr( + logging, log_level.value.upper() + ), # pylint: disable=no-member }, }, } From 06403e1376ea18cabee3681f65538cf3547a9257 Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 11:01:11 -0500 Subject: [PATCH 03/17] before superclass --- .../src/aqm_data_sync/aqm_data_sync_cli.py | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py index 35ad03c..8936734 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py @@ -9,9 +9,11 @@ app = typer.Typer(pretty_exceptions_enable=False) -@app.command() -def main( - dst_dir: Path = typer.Option(..., "--dst-dir", help="Destination directory for sync."), +@app.command(name="time-varying") +def time_varying( + dst_dir: Path = typer.Option( + ..., "--dst-dir", help="Destination directory for sync." + ), first_cycle_date: str = typer.Option( None, "--first-cycle-date", @@ -23,12 +25,9 @@ def main( "--last-cycle-date", help="Last cycle date in yyyymmdd format. If not provided, defaults to 24 hours after --first-cycle-date.", ), - s3_root: str = typer.Option("s3://noaa-ufs-srw-pds/UFS-AQM", "--s3-root", help="S3 root path."), - max_concurrent_requests: int = typer.Option( - 3, "--max-concurrent-requests", help="Max concurrent requests." + use_case: UseCaseKey = typer.Option( + UseCaseKey.UNDEFINED, "--use-case", help="Use case." ), - dry_run: bool = typer.Option(False, "--dry-run", help="Dry run."), - use_case: UseCaseKey = typer.Option(UseCaseKey.UNDEFINED, "--use-case", help="Use case."), snippet: bool = typer.Option( False, "--snippet", @@ -53,5 +52,17 @@ def main( runner.run() +@app.command(name="srw-fixed") +def srw_fixed( + dst_dir: Path = typer.Option( + ..., "--dst-dir", help="Destination directory for sync." + ), + max_concurrent_requests: int = typer.Option( + 3, "--max-concurrent-requests", help="Max concurrent requests." + ), + dry_run: bool = typer.Option(False, "--dry-run", help="Dry run."), +) -> None: ... + + if __name__ == "__main__": app() From e59b6c26a2a9d718334a3140befbb647271d679f Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 11:17:59 -0500 Subject: [PATCH 04/17] before abstract context --- .../src/aqm_data_sync/aqm_data_sync_cli.py | 4 +-- .../aqm-data-sync/src/aqm_data_sync/core.py | 36 +++++++++++-------- .../aqm-data-sync/src/test/test_core.py | 6 ++-- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py index 8936734..9521205 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py @@ -3,7 +3,7 @@ import typer -from aqm_data_sync.core import UseCaseKey, Context, UseCase, S3SyncRunner +from aqm_data_sync.core import UseCaseKey, Context, UseCase, TimeVaryingSyncRunner os.environ["NO_COLOR"] = "1" app = typer.Typer(pretty_exceptions_enable=False) @@ -48,7 +48,7 @@ def time_varying( ctx = Context(**kwds) else: ctx = UseCase.from_key(use_case, **kwds) - runner = S3SyncRunner(ctx) + runner = TimeVaryingSyncRunner(ctx) runner.run() diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py index 213eb19..b87f89e 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py @@ -1,6 +1,7 @@ import datetime import logging import subprocess +from abc import ABC, abstractmethod from enum import unique, StrEnum from pathlib import Path from typing import Any @@ -90,7 +91,7 @@ def _initialize_model_(cls, values: dict) -> dict: return values -class S3SyncRunner: +class AbstractS3SyncRunner(ABC): def __init__(self, context: Context) -> None: self._ctx = context @@ -136,6 +137,26 @@ def _create_sync_cmd_(self) -> tuple[str, ...]: cmd.append(str(self._ctx.dst_dir)) return tuple(cmd) + @abstractmethod + def _update_include_templates_(self, cmd: list[str]) -> None: + pass + + def _handle_max_concurrent_request_reset_(self): + if self._ctx.system_max_concurrent_requests is not None: + LOGGER("resetting max_concurrent_requests") + subprocess.check_call( + ( + "aws", + "configure", + "set", + "default.s3.max_concurrent_requests", + str(self._ctx.system_max_concurrent_requests), + ) + ) + + +class TimeVaryingSyncRunner(AbstractS3SyncRunner): + def _update_include_templates_(self, cmd: list[str]) -> None: restart_cycle_date = self._ctx.first_cycle_date - datetime.timedelta(days=1) curr_cycle_date = self._ctx.first_cycle_date @@ -181,16 +202,3 @@ def _create_include_templates_for_cycle_date_( f"GEFS_Aerosol/{curr_cycle_date_str}/00/gfs.t00z.atmf{fhr:03}.nemsio" ] return include_templates - - def _handle_max_concurrent_request_reset_(self): - if self._ctx.system_max_concurrent_requests is not None: - LOGGER("resetting max_concurrent_requests") - subprocess.check_call( - ( - "aws", - "configure", - "set", - "default.s3.max_concurrent_requests", - str(self._ctx.system_max_concurrent_requests), - ) - ) diff --git a/python_utils/aqm-data-sync/src/test/test_core.py b/python_utils/aqm-data-sync/src/test/test_core.py index 9e36832..04091f7 100644 --- a/python_utils/aqm-data-sync/src/test/test_core.py +++ b/python_utils/aqm-data-sync/src/test/test_core.py @@ -2,7 +2,7 @@ from aqm_data_sync.core import ( Context, - S3SyncRunner, + TimeVaryingSyncRunner, UseCase, UseCaseKey, UseCaseAeromma, @@ -15,7 +15,7 @@ def test_happy_path(self, tmp_path: Path) -> None: """Test a dry run with a single forecast date.""" first_cycle_date = "2023060112" ctx = Context(first_cycle_date=first_cycle_date, dst_dir=tmp_path, dry_run=True) - runner = S3SyncRunner(ctx) + runner = TimeVaryingSyncRunner(ctx) runner.run() def test_create_sync_command(self, tmp_path: Path) -> None: @@ -29,7 +29,7 @@ def test_create_sync_command(self, tmp_path: Path) -> None: dst_dir=dst_dir, dry_run=True, ) - runner = S3SyncRunner(ctx) + runner = TimeVaryingSyncRunner(ctx) actual = runner._create_sync_cmd_() expected = ( "aws", From a156f0eba369a8d1466e5c9f6987cd40445d89fe Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 11:27:12 -0500 Subject: [PATCH 05/17] extracted context --- .../src/aqm_data_sync/aqm_data_sync_cli.py | 21 +++++++++--------- .../aqm-data-sync/src/aqm_data_sync/core.py | 22 +++++++++++-------- .../aqm-data-sync/src/test/test_core.py | 6 ++--- 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py index 9521205..5a7ea94 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py @@ -3,7 +3,12 @@ import typer -from aqm_data_sync.core import UseCaseKey, Context, UseCase, TimeVaryingSyncRunner +from aqm_data_sync.core import ( + UseCaseKey, + TimeVaryingContext, + UseCase, + TimeVaryingSyncRunner, +) os.environ["NO_COLOR"] = "1" app = typer.Typer(pretty_exceptions_enable=False) @@ -11,9 +16,7 @@ @app.command(name="time-varying") def time_varying( - dst_dir: Path = typer.Option( - ..., "--dst-dir", help="Destination directory for sync." - ), + dst_dir: Path = typer.Option(..., "--dst-dir", help="Destination directory for sync."), first_cycle_date: str = typer.Option( None, "--first-cycle-date", @@ -25,9 +28,7 @@ def time_varying( "--last-cycle-date", help="Last cycle date in yyyymmdd format. If not provided, defaults to 24 hours after --first-cycle-date.", ), - use_case: UseCaseKey = typer.Option( - UseCaseKey.UNDEFINED, "--use-case", help="Use case." - ), + use_case: UseCaseKey = typer.Option(UseCaseKey.UNDEFINED, "--use-case", help="Use case."), snippet: bool = typer.Option( False, "--snippet", @@ -45,7 +46,7 @@ def time_varying( snippet=snippet, ) if use_case == UseCaseKey.UNDEFINED: - ctx = Context(**kwds) + ctx = TimeVaryingContext(**kwds) else: ctx = UseCase.from_key(use_case, **kwds) runner = TimeVaryingSyncRunner(ctx) @@ -54,9 +55,7 @@ def time_varying( @app.command(name="srw-fixed") def srw_fixed( - dst_dir: Path = typer.Option( - ..., "--dst-dir", help="Destination directory for sync." - ), + dst_dir: Path = typer.Option(..., "--dst-dir", help="Destination directory for sync."), max_concurrent_requests: int = typer.Option( 3, "--max-concurrent-requests", help="Max concurrent requests." ), diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py index b87f89e..a1e64b6 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py @@ -17,16 +17,12 @@ class UseCaseKey(StrEnum): AEROMMA = "AEROMMA" -class Context(BaseModel): +class AbstractContext(ABC, BaseModel): model_config = {"frozen": True} - first_cycle_date: datetime.datetime dst_dir: Path - fcst_hr: int = 0 - last_cycle_date: datetime.datetime - s3_root: str = "s3://noaa-ufs-srw-pds/UFS-AQM" + s3_root: str = "s3://noaa-ufs-srw-pds" max_concurrent_requests: int | None = 3 dry_run: bool = False - snippet: bool = False @computed_field def system_max_concurrent_requests(self) -> int | None: @@ -40,6 +36,14 @@ def system_max_concurrent_requests(self) -> int | None: else: return int(raw_output) + +class TimeVaryingContext(AbstractContext): + first_cycle_date: datetime.datetime + fcst_hr: int = 0 + last_cycle_date: datetime.datetime + s3_root: str = "s3://noaa-ufs-srw-pds/UFS-AQM" + snippet: bool = False + @model_validator(mode="before") @classmethod def _initialize_model_(cls, values: dict) -> dict: @@ -51,13 +55,13 @@ def _initialize_model_(cls, values: dict) -> dict: return values @model_validator(mode="after") - def _finalize_model_(self) -> "Context": + def _finalize_model_(self) -> "TimeVaryingContext": if self.last_cycle_date < self.first_cycle_date: raise ValueError("last_cycle_date must be >= first_cycle_date") return self -class UseCase(Context): +class UseCase(TimeVaryingContext): key: UseCaseKey @classmethod @@ -93,7 +97,7 @@ def _initialize_model_(cls, values: dict) -> dict: class AbstractS3SyncRunner(ABC): - def __init__(self, context: Context) -> None: + def __init__(self, context: TimeVaryingContext) -> None: self._ctx = context def run(self) -> None: diff --git a/python_utils/aqm-data-sync/src/test/test_core.py b/python_utils/aqm-data-sync/src/test/test_core.py index 04091f7..97b1afb 100644 --- a/python_utils/aqm-data-sync/src/test/test_core.py +++ b/python_utils/aqm-data-sync/src/test/test_core.py @@ -1,7 +1,7 @@ from pathlib import Path from aqm_data_sync.core import ( - Context, + TimeVaryingContext, TimeVaryingSyncRunner, UseCase, UseCaseKey, @@ -14,7 +14,7 @@ class TestS3SyncRunner: def test_happy_path(self, tmp_path: Path) -> None: """Test a dry run with a single forecast date.""" first_cycle_date = "2023060112" - ctx = Context(first_cycle_date=first_cycle_date, dst_dir=tmp_path, dry_run=True) + ctx = TimeVaryingContext(first_cycle_date=first_cycle_date, dst_dir=tmp_path, dry_run=True) runner = TimeVaryingSyncRunner(ctx) runner.run() @@ -23,7 +23,7 @@ def test_create_sync_command(self, tmp_path: Path) -> None: first_cycle_date = "2023060112" last_cycle_date = "2023060212" dst_dir = tmp_path / "output-for-this-test" - ctx = Context( + ctx = TimeVaryingContext( first_cycle_date=first_cycle_date, last_cycle_date=last_cycle_date, dst_dir=dst_dir, From 9688495c78c2709c7191339b98e965586fffa6e6 Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 11:45:43 -0500 Subject: [PATCH 06/17] unit tests pass --- python_utils/aqm-data-sync/environment.yml | 2 ++ .../src/aqm_data_sync/aqm_data_sync_cli.py | 17 +++++++++++++---- .../src/test/test_aqm_data_sync.py | 1 + 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/python_utils/aqm-data-sync/environment.yml b/python_utils/aqm-data-sync/environment.yml index a4946fc..6599490 100644 --- a/python_utils/aqm-data-sync/environment.yml +++ b/python_utils/aqm-data-sync/environment.yml @@ -8,3 +8,5 @@ dependencies: - pytest - pytest-mock - awscli + - black + - mypy diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py index 5a7ea94..a0fc81c 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py @@ -16,7 +16,9 @@ @app.command(name="time-varying") def time_varying( - dst_dir: Path = typer.Option(..., "--dst-dir", help="Destination directory for sync."), + dst_dir: Path = typer.Option( + ..., "--dst-dir", help="Destination directory for sync." + ), first_cycle_date: str = typer.Option( None, "--first-cycle-date", @@ -28,7 +30,13 @@ def time_varying( "--last-cycle-date", help="Last cycle date in yyyymmdd format. If not provided, defaults to 24 hours after --first-cycle-date.", ), - use_case: UseCaseKey = typer.Option(UseCaseKey.UNDEFINED, "--use-case", help="Use case."), + use_case: UseCaseKey = typer.Option( + UseCaseKey.UNDEFINED, "--use-case", help="Use case." + ), + max_concurrent_requests: int = typer.Option( + 3, "--max-concurrent-requests", help="Max concurrent requests." + ), + dry_run: bool = typer.Option(False, "--dry-run", help="Dry run."), snippet: bool = typer.Option( False, "--snippet", @@ -40,7 +48,6 @@ def time_varying( dst_dir=dst_dir, fcst_hr=fcst_hr, last_cycle_date=last_cycle_date, - s3_root=s3_root, max_concurrent_requests=max_concurrent_requests, dry_run=dry_run, snippet=snippet, @@ -55,7 +62,9 @@ def time_varying( @app.command(name="srw-fixed") def srw_fixed( - dst_dir: Path = typer.Option(..., "--dst-dir", help="Destination directory for sync."), + dst_dir: Path = typer.Option( + ..., "--dst-dir", help="Destination directory for sync." + ), max_concurrent_requests: int = typer.Option( 3, "--max-concurrent-requests", help="Max concurrent requests." ), diff --git a/python_utils/aqm-data-sync/src/test/test_aqm_data_sync.py b/python_utils/aqm-data-sync/src/test/test_aqm_data_sync.py index 7ff1f9c..dc5f6d1 100644 --- a/python_utils/aqm-data-sync/src/test/test_aqm_data_sync.py +++ b/python_utils/aqm-data-sync/src/test/test_aqm_data_sync.py @@ -20,6 +20,7 @@ def test_use_case(tmp_path: Path) -> None: runner = CliRunner() args = [ + "time-varying", "--use-case", UseCaseKey.AEROMMA.value, "--dst-dir", From c9ff0829bbda3cc288f885ec275ef24842804025 Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 11:48:29 -0500 Subject: [PATCH 07/17] add help text --- .../aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py | 4 ++-- python_utils/aqm-data-sync/src/test/test_aqm_data_sync.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py index a0fc81c..72d0c48 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py @@ -14,7 +14,7 @@ app = typer.Typer(pretty_exceptions_enable=False) -@app.command(name="time-varying") +@app.command(name="time-varying", help="Download time varying input data for UFS-AQM.") def time_varying( dst_dir: Path = typer.Option( ..., "--dst-dir", help="Destination directory for sync." @@ -60,7 +60,7 @@ def time_varying( runner.run() -@app.command(name="srw-fixed") +@app.command(name="srw-fixed", help="Download SRW fixed data.") def srw_fixed( dst_dir: Path = typer.Option( ..., "--dst-dir", help="Destination directory for sync." diff --git a/python_utils/aqm-data-sync/src/test/test_aqm_data_sync.py b/python_utils/aqm-data-sync/src/test/test_aqm_data_sync.py index dc5f6d1..d0a0fde 100644 --- a/python_utils/aqm-data-sync/src/test/test_aqm_data_sync.py +++ b/python_utils/aqm-data-sync/src/test/test_aqm_data_sync.py @@ -12,10 +12,11 @@ def test_help() -> None: """Test that the help message can be displayed.""" os.environ["TERMINAL_WIDTH"] = "100" cli_path = Path(__file__).parent.parent / "aqm_data_sync" / "aqm_data_sync_cli.py" - subprocess.check_call(["python", str(cli_path), "--help"]) + for subcommand in ("time-varying", "srw-fixed"): + subprocess.check_call(["python", str(cli_path), subcommand, "--help"]) -def test_use_case(tmp_path: Path) -> None: +def test_time_varying_use_case(tmp_path: Path) -> None: """Test the use case pathway for a snippet.""" runner = CliRunner() From b69b6d6018736a112aada1bc6a44a4277236b3da Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 13:25:33 -0500 Subject: [PATCH 08/17] add srw-fixed download --- .../src/aqm_data_sync/aqm_data_sync_cli.py | 54 +++++++++++++++---- .../aqm-data-sync/src/aqm_data_sync/core.py | 31 +++++++++-- .../aqm-data-sync/src/test/test_core.py | 41 +++++++++++++- 3 files changed, 109 insertions(+), 17 deletions(-) diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py index 72d0c48..fa7b698 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py @@ -2,23 +2,45 @@ from pathlib import Path import typer +from pydantic import BaseModel from aqm_data_sync.core import ( UseCaseKey, TimeVaryingContext, UseCase, TimeVaryingSyncRunner, + SRWFixedContext, + SRWFixedSyncRunner, ) os.environ["NO_COLOR"] = "1" app = typer.Typer(pretty_exceptions_enable=False) +class HelpMessage(BaseModel): + dst_dir: str = "Destination directory for sync." + max_concurrent_requests: str = "Maximum number of concurrent requests." + dry_run: str = "Dry run. Nothing will be materially synchronized." + + +class DefaultValue(BaseModel): + max_concurrent_requests: int = 5 + + +class FlagName(BaseModel): + dst_dir: str = "--dst-dir" + dry_run: str = "--dry-run" + max_concurrent_requests: str = "--max-concurrent-requests" + + +_HELP = HelpMessage() +_DEFAULT = DefaultValue() +_FLAG_NAME = FlagName() + + @app.command(name="time-varying", help="Download time varying input data for UFS-AQM.") def time_varying( - dst_dir: Path = typer.Option( - ..., "--dst-dir", help="Destination directory for sync." - ), + dst_dir: Path = typer.Option(..., _FLAG_NAME.dst_dir, help=_HELP.dst_dir), first_cycle_date: str = typer.Option( None, "--first-cycle-date", @@ -34,9 +56,11 @@ def time_varying( UseCaseKey.UNDEFINED, "--use-case", help="Use case." ), max_concurrent_requests: int = typer.Option( - 3, "--max-concurrent-requests", help="Max concurrent requests." + _DEFAULT.max_concurrent_requests, + _FLAG_NAME.max_concurrent_requests, + help=_HELP.max_concurrent_requests, ), - dry_run: bool = typer.Option(False, "--dry-run", help="Dry run."), + dry_run: bool = typer.Option(False, _FLAG_NAME.dry_run, help=_HELP.dry_run), snippet: bool = typer.Option( False, "--snippet", @@ -62,14 +86,22 @@ def time_varying( @app.command(name="srw-fixed", help="Download SRW fixed data.") def srw_fixed( - dst_dir: Path = typer.Option( - ..., "--dst-dir", help="Destination directory for sync." - ), + dst_dir: Path = typer.Option(..., _FLAG_NAME.dst_dir, help=_HELP.dst_dir), max_concurrent_requests: int = typer.Option( - 3, "--max-concurrent-requests", help="Max concurrent requests." + _DEFAULT.max_concurrent_requests, + _FLAG_NAME.max_concurrent_requests, + help=_HELP.max_concurrent_requests, ), - dry_run: bool = typer.Option(False, "--dry-run", help="Dry run."), -) -> None: ... + dry_run: bool = typer.Option(False, _FLAG_NAME.dry_run, help=_HELP.dry_run), +) -> None: + kwds = dict( + dst_dir=dst_dir, + max_concurrent_requests=max_concurrent_requests, + dry_run=dry_run, + ) + ctx = SRWFixedContext(**kwds) + runner = SRWFixedSyncRunner(ctx) + runner.run() if __name__ == "__main__": diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py index a1e64b6..20ecb9a 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py @@ -37,6 +37,10 @@ def system_max_concurrent_requests(self) -> int | None: return int(raw_output) +class SRWFixedContext(AbstractContext): + s3_root: str = "s3://noaa-ufs-srw-pds/develop-20250702/fix" + + class TimeVaryingContext(AbstractContext): first_cycle_date: datetime.datetime fcst_hr: int = 0 @@ -97,7 +101,7 @@ def _initialize_model_(cls, values: dict) -> dict: class AbstractS3SyncRunner(ABC): - def __init__(self, context: TimeVaryingContext) -> None: + def __init__(self, context: AbstractContext) -> None: self._ctx = context def run(self) -> None: @@ -159,8 +163,20 @@ def _handle_max_concurrent_request_reset_(self): ) +class SRWFixedSyncRunner(AbstractS3SyncRunner): + + def __init__(self, context: SRWFixedContext) -> None: + super().__init__(context) + + def _update_include_templates_(self, cmd: list[str]) -> None: + cmd += ["--include", "*"] + + class TimeVaryingSyncRunner(AbstractS3SyncRunner): + def __init__(self, context: TimeVaryingContext) -> None: + super().__init__(context) + def _update_include_templates_(self, cmd: list[str]) -> None: restart_cycle_date = self._ctx.first_cycle_date - datetime.timedelta(days=1) curr_cycle_date = self._ctx.first_cycle_date @@ -169,13 +185,20 @@ def _update_include_templates_(self, cmd: list[str]) -> None: LOGGER(f"{ctr=}, {curr_cycle_date=}") if ctr > 1000: LOGGER("", exc_info=ValueError(f"{ctr=} - Exceeded max iterations")) - include_templates = self._create_include_templates_for_cycle_date_(curr_cycle_date) + include_templates = self._create_include_templates_for_cycle_date_( + curr_cycle_date + ) if ctr == 0: LOGGER("adding restart file download") - include_templates.append(f"RESTART/*{restart_cycle_date.strftime('%Y%m%d')}*") + include_templates.append( + f"RESTART/*{restart_cycle_date.strftime('%Y%m%d')}*" + ) for it in include_templates: cmd += ["--include", it] - if curr_cycle_date == self._ctx.last_cycle_date or self._ctx.snippet is True: + if ( + curr_cycle_date == self._ctx.last_cycle_date + or self._ctx.snippet is True + ): LOGGER("finished adding include filters") break curr_cycle_date += datetime.timedelta(days=1) diff --git a/python_utils/aqm-data-sync/src/test/test_core.py b/python_utils/aqm-data-sync/src/test/test_core.py index 97b1afb..7c1d7b2 100644 --- a/python_utils/aqm-data-sync/src/test/test_core.py +++ b/python_utils/aqm-data-sync/src/test/test_core.py @@ -6,15 +6,19 @@ UseCase, UseCaseKey, UseCaseAeromma, + SRWFixedContext, + SRWFixedSyncRunner, ) -class TestS3SyncRunner: +class TestTimeVaryingSyncRunner: def test_happy_path(self, tmp_path: Path) -> None: """Test a dry run with a single forecast date.""" first_cycle_date = "2023060112" - ctx = TimeVaryingContext(first_cycle_date=first_cycle_date, dst_dir=tmp_path, dry_run=True) + ctx = TimeVaryingContext( + first_cycle_date=first_cycle_date, dst_dir=tmp_path, dry_run=True + ) runner = TimeVaryingSyncRunner(ctx) runner.run() @@ -176,3 +180,36 @@ def test_from_key(self, tmp_path: Path) -> None: use_case = UseCase.from_key(UseCaseKey.AEROMMA, dst_dir=tmp_path) print(use_case) assert isinstance(use_case, UseCaseAeromma) + + +class TestSRWFixedSyncRunner: + + def test_create_sync_command(self, tmp_path: Path) -> None: + """Test an exact match for the AWS S3 sync command.""" + dst_dir = tmp_path / "output-for-this-test" + ctx = SRWFixedContext( + dst_dir=dst_dir, + dry_run=True, + ) + runner = SRWFixedSyncRunner(ctx) + actual = runner._create_sync_cmd_() + expected = ( + "aws", + "s3", + "sync", + "--no-sign-request", + "--dryrun", + "--exclude", + "*", + "--include", + "*", + "s3://noaa-ufs-srw-pds/develop-20250702/fix", + str(dst_dir), + ) + try: + assert actual == expected + except AssertionError: + print(f"{actual=}") + diff = set(actual).symmetric_difference(set(expected)) + print(f"{diff=}") + raise From 11c4515936dd05ec2812aac700ae0952c19d18be Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 13:32:14 -0500 Subject: [PATCH 09/17] before fixing context typing --- .../src/aqm_data_sync/logging_aqm_data_sync.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/logging_aqm_data_sync.py b/python_utils/aqm-data-sync/src/aqm_data_sync/logging_aqm_data_sync.py index 74a6398..ec487b8 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/logging_aqm_data_sync.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/logging_aqm_data_sync.py @@ -36,7 +36,7 @@ def __call__( """ if exc_info is not None: level = logging.ERROR - self.logger.log(level, msg, exc_info=exc_info, stacklevel=stacklevel) + self._get_logger_().log(level, msg, exc_info=exc_info, stacklevel=stacklevel) if exc_info is not None and self.exit_on_error: raise exc_info @@ -84,6 +84,11 @@ def initialize( self.logger = logging.getLogger(_PROJECT_NAME) self("logging initialized") + def _get_logger_(self) -> logging.Logger: + if self.logger is None: + raise ValueError + return self.logger + LOGGER = LoggerWrapper() LOGGER.initialize(log_level=LogLevel.DEBUG) From 548c32a5f4e085422d251bdaccffb4e4d361ef79 Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 13:56:07 -0500 Subject: [PATCH 10/17] before generic type --- .../aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py | 4 ++-- python_utils/aqm-data-sync/src/aqm_data_sync/core.py | 8 +------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py index fa7b698..759dab6 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/aqm_data_sync_cli.py @@ -77,7 +77,7 @@ def time_varying( snippet=snippet, ) if use_case == UseCaseKey.UNDEFINED: - ctx = TimeVaryingContext(**kwds) + ctx = TimeVaryingContext.model_validate(kwds) else: ctx = UseCase.from_key(use_case, **kwds) runner = TimeVaryingSyncRunner(ctx) @@ -99,7 +99,7 @@ def srw_fixed( max_concurrent_requests=max_concurrent_requests, dry_run=dry_run, ) - ctx = SRWFixedContext(**kwds) + ctx = SRWFixedContext.model_validate(kwds) runner = SRWFixedSyncRunner(ctx) runner.run() diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py index 20ecb9a..6e00f79 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py @@ -3,6 +3,7 @@ import subprocess from abc import ABC, abstractmethod from enum import unique, StrEnum +from functools import cached_property from pathlib import Path from typing import Any @@ -165,18 +166,12 @@ def _handle_max_concurrent_request_reset_(self): class SRWFixedSyncRunner(AbstractS3SyncRunner): - def __init__(self, context: SRWFixedContext) -> None: - super().__init__(context) - def _update_include_templates_(self, cmd: list[str]) -> None: cmd += ["--include", "*"] class TimeVaryingSyncRunner(AbstractS3SyncRunner): - def __init__(self, context: TimeVaryingContext) -> None: - super().__init__(context) - def _update_include_templates_(self, cmd: list[str]) -> None: restart_cycle_date = self._ctx.first_cycle_date - datetime.timedelta(days=1) curr_cycle_date = self._ctx.first_cycle_date @@ -203,7 +198,6 @@ def _update_include_templates_(self, cmd: list[str]) -> None: break curr_cycle_date += datetime.timedelta(days=1) ctr += 1 - return cmd def _create_include_templates_for_cycle_date_( self, curr_cycle_date: datetime.datetime From b66ffd44d75f374dde9aefafb7bb334caa925d7b Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 14:06:47 -0500 Subject: [PATCH 11/17] mypy passing --- .../aqm-data-sync/src/aqm_data_sync/core.py | 14 ++++++++------ python_utils/aqm-data-sync/src/test/test_core.py | 16 +++++++++------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py index 6e00f79..a226975 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py @@ -3,9 +3,8 @@ import subprocess from abc import ABC, abstractmethod from enum import unique, StrEnum -from functools import cached_property from pathlib import Path -from typing import Any +from typing import Any, TypeVar, Generic from pydantic import BaseModel, computed_field, model_validator @@ -100,9 +99,12 @@ def _initialize_model_(cls, values: dict) -> dict: return values -class AbstractS3SyncRunner(ABC): +T = TypeVar("T", bound=AbstractContext) - def __init__(self, context: AbstractContext) -> None: + +class AbstractS3SyncRunner(ABC, Generic[T]): + + def __init__(self, context: T) -> None: self._ctx = context def run(self) -> None: @@ -164,13 +166,13 @@ def _handle_max_concurrent_request_reset_(self): ) -class SRWFixedSyncRunner(AbstractS3SyncRunner): +class SRWFixedSyncRunner(AbstractS3SyncRunner[SRWFixedContext]): def _update_include_templates_(self, cmd: list[str]) -> None: cmd += ["--include", "*"] -class TimeVaryingSyncRunner(AbstractS3SyncRunner): +class TimeVaryingSyncRunner(AbstractS3SyncRunner[TimeVaryingContext]): def _update_include_templates_(self, cmd: list[str]) -> None: restart_cycle_date = self._ctx.first_cycle_date - datetime.timedelta(days=1) diff --git a/python_utils/aqm-data-sync/src/test/test_core.py b/python_utils/aqm-data-sync/src/test/test_core.py index 7c1d7b2..71c82a7 100644 --- a/python_utils/aqm-data-sync/src/test/test_core.py +++ b/python_utils/aqm-data-sync/src/test/test_core.py @@ -16,8 +16,8 @@ class TestTimeVaryingSyncRunner: def test_happy_path(self, tmp_path: Path) -> None: """Test a dry run with a single forecast date.""" first_cycle_date = "2023060112" - ctx = TimeVaryingContext( - first_cycle_date=first_cycle_date, dst_dir=tmp_path, dry_run=True + ctx = TimeVaryingContext.model_validate( + dict(first_cycle_date=first_cycle_date, dst_dir=tmp_path, dry_run=True) ) runner = TimeVaryingSyncRunner(ctx) runner.run() @@ -27,11 +27,13 @@ def test_create_sync_command(self, tmp_path: Path) -> None: first_cycle_date = "2023060112" last_cycle_date = "2023060212" dst_dir = tmp_path / "output-for-this-test" - ctx = TimeVaryingContext( - first_cycle_date=first_cycle_date, - last_cycle_date=last_cycle_date, - dst_dir=dst_dir, - dry_run=True, + ctx = TimeVaryingContext.model_validate( + dict( + first_cycle_date=first_cycle_date, + last_cycle_date=last_cycle_date, + dst_dir=dst_dir, + dry_run=True, + ) ) runner = TimeVaryingSyncRunner(ctx) actual = runner._create_sync_cmd_() From 7530df76b4fa9cb2732b9f474dbd5f14d68f901a Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 14:08:31 -0500 Subject: [PATCH 12/17] mypy passing --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 97e7957..73532e3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ __pycache__/ *.pyo *.pyd .pytest_cache/ -*.idea \ No newline at end of file +*.idea +*.egg-info/ From 4c66f8d819c9f00cdafbb274a301665aad207987 Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 14:16:10 -0500 Subject: [PATCH 13/17] mypy passing --- python_utils/aqm-data-sync/src/aqm_data_sync/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py index a226975..8f36bd5 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py @@ -38,7 +38,7 @@ def system_max_concurrent_requests(self) -> int | None: class SRWFixedContext(AbstractContext): - s3_root: str = "s3://noaa-ufs-srw-pds/develop-20250702/fix" + s3_root: str = "s3://noaa-ufs-srw-pds" class TimeVaryingContext(AbstractContext): @@ -169,7 +169,7 @@ def _handle_max_concurrent_request_reset_(self): class SRWFixedSyncRunner(AbstractS3SyncRunner[SRWFixedContext]): def _update_include_templates_(self, cmd: list[str]) -> None: - cmd += ["--include", "*"] + cmd += ["--include", "develop-20250702/fix/*"] class TimeVaryingSyncRunner(AbstractS3SyncRunner[TimeVaryingContext]): From 67cd6022d3331acc2108d7a44a33615b8e50edd3 Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 14:19:02 -0500 Subject: [PATCH 14/17] fix test --- python_utils/aqm-data-sync/src/test/test_core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_utils/aqm-data-sync/src/test/test_core.py b/python_utils/aqm-data-sync/src/test/test_core.py index 71c82a7..3eaca4d 100644 --- a/python_utils/aqm-data-sync/src/test/test_core.py +++ b/python_utils/aqm-data-sync/src/test/test_core.py @@ -204,8 +204,8 @@ def test_create_sync_command(self, tmp_path: Path) -> None: "--exclude", "*", "--include", - "*", - "s3://noaa-ufs-srw-pds/develop-20250702/fix", + "develop-20250702/fix/*", + "s3://noaa-ufs-srw-pds", str(dst_dir), ) try: From 7b96f6a466481170990efaf0d97825655a3f75ab Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 14:22:44 -0500 Subject: [PATCH 15/17] add pre-commit --- python_utils/aqm-data-sync/pre-commit.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 python_utils/aqm-data-sync/pre-commit.py diff --git a/python_utils/aqm-data-sync/pre-commit.py b/python_utils/aqm-data-sync/pre-commit.py new file mode 100644 index 0000000..0279d7a --- /dev/null +++ b/python_utils/aqm-data-sync/pre-commit.py @@ -0,0 +1,6 @@ +import subprocess + + +subprocess.check_call(["black", "src"]) +subprocess.check_call(["mypy", "src"]) +subprocess.check_call(["pytest", "src/test"]) From 9153cafc82820142cfdad664577cc1275a6cd1cc Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Thu, 7 Aug 2025 15:01:33 -0500 Subject: [PATCH 16/17] update readme, change fix pathing --- python_utils/aqm-data-sync/README.md | 41 ++++++++++++------- .../aqm-data-sync/src/aqm_data_sync/core.py | 4 +- .../src/test/test_aqm_data_sync.py | 1 + .../aqm-data-sync/src/test/test_core.py | 4 +- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/python_utils/aqm-data-sync/README.md b/python_utils/aqm-data-sync/README.md index c1c345f..9b58e52 100644 --- a/python_utils/aqm-data-sync/README.md +++ b/python_utils/aqm-data-sync/README.md @@ -20,11 +20,18 @@ pytest test # Usage ```shell -conda run -n aqm-data-sync aqm-data-sync --help +Usage: aqm_data_sync_cli.py [OPTIONS] COMMAND [ARGS]... -Usage: aqm-data-sync [OPTIONS] +┌─ Commands ───────────────────────────────────────────────────────────────────────────────────────┐ +│ time-varying Download time varying input data for UFS-AQM. │ +│ srw-fixed Download SRW fixed data. │ +└──────────────────────────────────────────────────────────────────────────────────────────────────┘ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ + Usage: aqm_data_sync_cli.py time-varying [OPTIONS] + + Download time varying input data for UFS-AQM. + +┌─ Options ────────────────────────────────────────────────────────────────────────────────────────┐ │ * --dst-dir PATH Destination directory for sync. │ │ [default: None] │ │ [required] │ @@ -36,19 +43,25 @@ Usage: aqm-data-sync [OPTIONS] │ not provided, defaults to 24 hours │ │ after --first-cycle-date. │ │ [default: None] │ -│ --s3-root TEXT S3 root path. │ -│ [default: │ -│ s3://noaa-ufs-srw-pds/UFS-AQM] │ -│ --max-concurrent-requests INTEGER Max concurrent requests. [default: 3] │ -│ --dry-run Dry run. │ │ --use-case [UNDEFINED|AEROMMA] Use case. [default: UNDEFINED] │ +│ --max-concurrent-requests INTEGER Maximum number of concurrent requests. │ +│ [default: 5] │ +│ --dry-run Dry run. Nothing will be materially │ +│ synchronized. │ │ --snippet If provided, download data for a single │ │ forecast cycle loop (e.g. one day). │ -│ --install-completion Install completion for the current │ -│ shell. │ -│ --show-completion Show completion for the current shell, │ -│ to copy it or customize the │ -│ installation. │ │ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +└──────────────────────────────────────────────────────────────────────────────────────────────────┘ + + Usage: aqm_data_sync_cli.py srw-fixed [OPTIONS] + + Download SRW fixed data. + +┌─ Options ────────────────────────────────────────────────────────────────────────────────────────┐ +│ * --dst-dir PATH Destination directory for sync. [default: None] │ +│ [required] │ +│ --max-concurrent-requests INTEGER Maximum number of concurrent requests. [default: 5] │ +│ --dry-run Dry run. Nothing will be materially synchronized. │ +│ --help Show this message and exit. │ +└──────────────────────────────────────────────────────────────────────────────────────────────────┘ ``` \ No newline at end of file diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py index 8f36bd5..860bfc7 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py @@ -38,7 +38,7 @@ def system_max_concurrent_requests(self) -> int | None: class SRWFixedContext(AbstractContext): - s3_root: str = "s3://noaa-ufs-srw-pds" + s3_root: str = "s3://noaa-ufs-srw-pds/develop-20250702" class TimeVaryingContext(AbstractContext): @@ -169,7 +169,7 @@ def _handle_max_concurrent_request_reset_(self): class SRWFixedSyncRunner(AbstractS3SyncRunner[SRWFixedContext]): def _update_include_templates_(self, cmd: list[str]) -> None: - cmd += ["--include", "develop-20250702/fix/*"] + cmd += ["--include", "fix/*"] class TimeVaryingSyncRunner(AbstractS3SyncRunner[TimeVaryingContext]): diff --git a/python_utils/aqm-data-sync/src/test/test_aqm_data_sync.py b/python_utils/aqm-data-sync/src/test/test_aqm_data_sync.py index d0a0fde..bf8d0ef 100644 --- a/python_utils/aqm-data-sync/src/test/test_aqm_data_sync.py +++ b/python_utils/aqm-data-sync/src/test/test_aqm_data_sync.py @@ -12,6 +12,7 @@ def test_help() -> None: """Test that the help message can be displayed.""" os.environ["TERMINAL_WIDTH"] = "100" cli_path = Path(__file__).parent.parent / "aqm_data_sync" / "aqm_data_sync_cli.py" + subprocess.check_call(["python", str(cli_path), "--help"]) for subcommand in ("time-varying", "srw-fixed"): subprocess.check_call(["python", str(cli_path), subcommand, "--help"]) diff --git a/python_utils/aqm-data-sync/src/test/test_core.py b/python_utils/aqm-data-sync/src/test/test_core.py index 3eaca4d..eb7b2d9 100644 --- a/python_utils/aqm-data-sync/src/test/test_core.py +++ b/python_utils/aqm-data-sync/src/test/test_core.py @@ -204,8 +204,8 @@ def test_create_sync_command(self, tmp_path: Path) -> None: "--exclude", "*", "--include", - "develop-20250702/fix/*", - "s3://noaa-ufs-srw-pds", + "fix/*", + "s3://noaa-ufs-srw-pds/develop-20250702", str(dst_dir), ) try: From 244d51a89d89a68fe54d443d25b0e3931cc280c4 Mon Sep 17 00:00:00 2001 From: Ben Koziol Date: Mon, 11 Aug 2025 17:01:17 -0600 Subject: [PATCH 17/17] download naturalearth shapefiles --- python_utils/aqm-data-sync/src/aqm_data_sync/core.py | 2 +- python_utils/aqm-data-sync/src/test/test_core.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py index 860bfc7..3c9d2db 100644 --- a/python_utils/aqm-data-sync/src/aqm_data_sync/core.py +++ b/python_utils/aqm-data-sync/src/aqm_data_sync/core.py @@ -169,7 +169,7 @@ def _handle_max_concurrent_request_reset_(self): class SRWFixedSyncRunner(AbstractS3SyncRunner[SRWFixedContext]): def _update_include_templates_(self, cmd: list[str]) -> None: - cmd += ["--include", "fix/*"] + cmd += ["--include", "fix/*", "--include", "NaturalEarth/*"] class TimeVaryingSyncRunner(AbstractS3SyncRunner[TimeVaryingContext]): diff --git a/python_utils/aqm-data-sync/src/test/test_core.py b/python_utils/aqm-data-sync/src/test/test_core.py index eb7b2d9..a1d041b 100644 --- a/python_utils/aqm-data-sync/src/test/test_core.py +++ b/python_utils/aqm-data-sync/src/test/test_core.py @@ -205,6 +205,8 @@ def test_create_sync_command(self, tmp_path: Path) -> None: "*", "--include", "fix/*", + "--include", + "NaturalEarth/*", "s3://noaa-ufs-srw-pds/develop-20250702", str(dst_dir), )