From 6b03299c6cede2067a709e90afb1120690d09bad Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Fri, 6 Jun 2025 11:12:00 -0600 Subject: [PATCH 001/150] sima updates to get rrtmgp lw to generate caps --- .gitmodules | 6 +- ccpp_framework | 2 +- cime_config/cam_autogen.py | 8 + cime_config/namelist_definition_cam.xml | 62 ++++++++ src/control/cam_comp.F90 | 6 +- src/control/runtime_obj.F90 | 8 +- src/control/runtime_obj.meta | 12 ++ src/data/generate_registry_data.py | 7 +- src/data/registry.xml | 175 ++++++++++++++++++++++ src/physics/ncar_ccpp | 2 +- src/physics/utils/orbital_data.F90 | 26 +++- src/physics/utils/orbital_data.meta | 6 + src/physics/utils/radiation_namelist.F90 | 133 ++++++++++++++++ src/physics/utils/radiation_namelist.meta | 43 ++++++ 14 files changed, 485 insertions(+), 11 deletions(-) create mode 100644 src/physics/utils/radiation_namelist.F90 create mode 100644 src/physics/utils/radiation_namelist.meta diff --git a/.gitmodules b/.gitmodules index e11a5553..4198073d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "ccpp-framework"] path = ccpp_framework url = https://github.com/NCAR/ccpp-framework - fxtag = 2025-01-06-dev + fxtag = 2025-06-03-dev fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework [submodule "history"] @@ -19,8 +19,8 @@ fxDONOTUSEurl = https://github.com/MPAS-Dev/MPAS-Model.git [submodule "ncar-physics"] path = src/physics/ncar_ccpp - url = https://github.com/ESCOMP/atmospheric_physics - fxtag = bb7c61028c1d6ead2c09b2c58cfbbfb3c3638a51 + url = https://github.com/peverwhee/atmospheric_physics + fxtag = 61aea3a46db719b0dbfad31ecf47978f8df64e6b fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/ccpp_framework b/ccpp_framework index fca3a9e3..9254784f 160000 --- a/ccpp_framework +++ b/ccpp_framework @@ -1 +1 @@ -Subproject commit fca3a9e3df101bd07a228631e5509ec4a93a9906 +Subproject commit 9254784f55757cdc8386509fedc7e6fdd02517ab diff --git a/cime_config/cam_autogen.py b/cime_config/cam_autogen.py index 695364b2..560f8201 100644 --- a/cime_config/cam_autogen.py +++ b/cime_config/cam_autogen.py @@ -489,6 +489,14 @@ def generate_physics_suites(build_cache, preproc_defs, host_name, # End if # End for # End for + + # Reorder scheme files to put namelist files first + namelist_files = [f for f in scheme_files if "_namelist" in f] + other_files = [f for f in scheme_files if "_namelist" not in f] + + # Concatenate with namelist files first + scheme_files = namelist_files + other_files + # Figure out if we need to generate new physics code genccpp_dir = os.path.join(bldroot, "ccpp") kind_phys = ['kind_phys = REAL64'] diff --git a/cime_config/namelist_definition_cam.xml b/cime_config/namelist_definition_cam.xml index 26cfa2d8..27bb26eb 100644 --- a/cime_config/namelist_definition_cam.xml +++ b/cime_config/namelist_definition_cam.xml @@ -394,6 +394,68 @@ + + + + integer + radiation + radiation_nl + + The frequency at which shortwave calculation is performed. + positive: time steps; negative: hours + + + -1 + + + + integer + radiation + radiation_nl + + The frequency at which longwave calculation is performed. + positive: time steps; negative: hours + + + -1 + + + + integer + radiation + radiation_nl + + The amount of time that radiation (both SW and LW) is run continuously from the start of an initial or restart run + positive: time steps; negative: hours + + + -1 + + + + logical + radiation + radiation_nl + + If true, use the namelist-defined radiation uniform angle in the solar zenith angle calculation + + + .false. + + + + real + kind_phys + radiation + radiation_nl + + The radiation timestep size for the solar zenith angle calculation + + + -99 + + + diff --git a/src/control/cam_comp.F90 b/src/control/cam_comp.F90 index 16e32984..8912d93e 100644 --- a/src/control/cam_comp.F90 +++ b/src/control/cam_comp.F90 @@ -190,7 +190,9 @@ subroutine cam_init(caseid, ctitle, model_doi_url, & ! Get current fractional calendar day. Needs to be updated at every timestep. calday = get_curr_calday() + next_calday = get_curr_calday(offset=int(get_step_size())) call mark_as_initialized('fractional_calendar_days_on_end_of_current_timestep') + call mark_as_initialized('fractional_calendar_days_on_end_of_next_timestep') ! Read CAM namelists. filein = "atm_in" // trim(inst_suffix) @@ -299,12 +301,14 @@ subroutine cam_timestep_init() use physics_grid, only: lat_rad, lon_rad use orbital_data, only: orbital_data_advance use stepon, only: stepon_timestep_init + use physics_types,only: dt_avg ! Update current fractional calendar day. Needs to be updated at every timestep. calday = get_curr_calday() ! Update the orbital data - call orbital_data_advance(calday, lat_rad, lon_rad) + call orbital_data_advance(calday, lat_rad, lon_rad, use_rad_uniform_angle, & + rad_uniform_angle, dt_avg) ! Update timestep flags in physics state is_first_timestep = is_first_step() diff --git a/src/control/runtime_obj.F90 b/src/control/runtime_obj.F90 index 157545a8..0879f5f3 100644 --- a/src/control/runtime_obj.F90 +++ b/src/control/runtime_obj.F90 @@ -2,12 +2,18 @@ module runtime_obj use shr_kind_mod, only: CS => SHR_KIND_CS use shr_kind_mod, only: r8=>shr_kind_r8 + use ccpp_kinds, only: kind_phys implicit none private + !> \section arg_table_runtime_obj Argument Table + !! \htmlinclude arg_table_runtime_obj.html + !! + real(kind_phys), public, parameter :: unset_real = huge(1.0_r8) + character(len=*), public, parameter :: unset_str = 'UNSET' integer, public, parameter :: unset_int = huge(1) - real(r8), public, parameter :: unset_real = huge(1.0_r8) + !real(r8), public, parameter :: unset_real = huge(1.0_r8) ! Water vapor constituent standard name character(len=*), public, parameter :: wv_stdname = 'water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water' diff --git a/src/control/runtime_obj.meta b/src/control/runtime_obj.meta index 6379f135..e7f5d0da 100644 --- a/src/control/runtime_obj.meta +++ b/src/control/runtime_obj.meta @@ -1,3 +1,15 @@ +[ccpp-table-properties] + name = runtime_obj + type = module +[ccpp-arg-table] + name = runtime_obj + type = module +[ unset_real ] + standard_name = definition_of_unset_for_real_variables + units = 1 + type = real | kind = kind_phys + protected = True + dimensions = () [ccpp-table-properties] name = runtime_options type = ddt diff --git a/src/data/generate_registry_data.py b/src/data/generate_registry_data.py index f4bcc10b..2e2c18fe 100755 --- a/src/data/generate_registry_data.py +++ b/src/data/generate_registry_data.py @@ -1064,11 +1064,12 @@ def write_definition(self, outfile, access, indent): # end for - def check_initial_values(self, physconst_vars): + def check_initial_values(self, physconst_vars,use_statements): """Raise an error if there are any initial values that are set to non-"used" and/or non-"physconst" variables""" + print(use_statements) for var in self.known_initial_value_vars: - if var not in physconst_vars: + if var not in physconst_vars and not any(second == var for _, second in use_statements): emsg = f"Initial value '{var}' is not a physconst variable" emsg += " or does not have necessary use statement" raise CCPPError(emsg) @@ -1695,7 +1696,7 @@ def write_registry_files(registry, dycore, outdir, src_mod, src_root, # end for # Then check against the initial values in the variable dictionary # Check will raise an exception if there is a rogue variable - file_.var_dict.check_initial_values(physconst_vars) + file_.var_dict.check_initial_values(physconst_vars,file_.use_statements) # Generate metadata and source if file_.generate_code: file_.write_metadata(outdir, logger) diff --git a/src/data/registry.xml b/src/data/registry.xml index e0de2464..5fcfca16 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -3,6 +3,12 @@ + $SRCROOT/src/physics/ncar_ccpp/schemes/rrtmgp/objects/ccpp_fluxes_byband.meta + $SRCROOT/src/physics/ncar_ccpp/schemes/rrtmgp/objects/ccpp_fluxes.meta + $SRCROOT/src/physics/ncar_ccpp/schemes/rrtmgp/objects/ccpp_gas_concentrations.meta + $SRCROOT/src/physics/ncar_ccpp/schemes/rrtmgp/objects/ccpp_gas_optics_rrtmgp.meta + $SRCROOT/src/physics/ncar_ccpp/schemes/rrtmgp/objects/ccpp_optical_props.meta + $SRCROOT/src/physics/ncar_ccpp/schemes/rrtmgp/objects/ccpp_source_functions.meta $SRCROOT/src/utils/spmd_utils.meta $SRCROOT/src/control/cam_control_mod.meta $SRCROOT/src/control/cam_logfile.meta @@ -12,6 +18,7 @@ $SRCROOT/src/physics/utils/orbital_data.meta $SRCROOT/src/physics/utils/musica_ccpp_dependencies.meta $SRCROOT/src/physics/utils/physics_grid.meta + $SRCROOT/src/physics/utils/radiation_namelist.meta $SRCROOT/src/physics/utils/cam_constituents.meta $SRCROOT/src/physics/utils/tropopause_climo_read.meta $SRCROOT/src/data/air_composition.meta @@ -25,6 +32,7 @@ + @@ -460,6 +468,11 @@ units="1" type="real" kind="kind_phys"> fractional calendar day at end of current timestep + + fractional calendar day at end of next timestep + .true. .false. + + + 0._kind_phys + + + horizontal_dimension vertical_layer_dimension + unset_real + CLDFSNOW pbuf_CLDFSNOW + + + horizontal_dimension vertical_layer_dimension + unset_real + CLDFGRAU pbuf_CLDFGRAU + + + horizontal_dimension vertical_layer_dimension + unset_real + ICGRAUWP pbuf_ICGRAUWP + + + horizontal_dimension vertical_layer_dimension + unset_real + DEGRAU pbuf_DEGRAU + + + horizontal_dimension vertical_layer_dimension + LAMBDAC pbuf_LAMBDAC + + + horizontal_dimension vertical_layer_dimension + MU pbuf_MU + + + horizontal_dimension vertical_layer_dimension + ICLWP pbuf_ICLWP + + + horizontal_dimension vertical_layer_dimension + ICIWP pbuf_ICIWP + + + horizontal_dimension vertical_layer_dimension + ICSWP pbuf_ICSWP + + + horizontal_dimension vertical_layer_dimension + DEI pbuf_DEI + + + horizontal_dimension vertical_layer_dimension + DES pbuf_DES + + + -1 + + + horizontal_dimension vertical_layer_dimension + QRS pbuf_QRS + + + horizontal_dimension vertical_layer_dimension + QRL pbuf_QRL + + + 2 + + + -1 + + + -1 + + + -1 + + + + horizontal_dimension + lwup cam_in_lwup + + + horizontal_dimension + asdir cam_in_asdir + + + horizontal_dimension + asdif cam_in_asdif + + + horizontal_dimension + aldir cam_in_aldir + + + horizontal_dimension + aldif cam_in_aldif + diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index bb7c6102..61aea3a4 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit bb7c61028c1d6ead2c09b2c58cfbbfb3c3638a51 +Subproject commit 61aea3a46db719b0dbfad31ecf47978f8df64e6b diff --git a/src/physics/utils/orbital_data.F90 b/src/physics/utils/orbital_data.F90 index bced4e1e..9d18fbd0 100644 --- a/src/physics/utils/orbital_data.F90 +++ b/src/physics/utils/orbital_data.F90 @@ -21,6 +21,7 @@ module orbital_data real(kind_phys), protected, public :: solar_declination = FILL_R8 ! Solar declination angle [radians] real(kind_phys), protected, public :: earth_sun_distance = FILL_R8 ! Earth-sun distance [AU] real(kind_phys), allocatable, protected, public :: solar_zenith_angle(:) ! Solar zenith angle (column) [radians] + real(kind_phys), allocatable, protected, public :: coszrs_rad(:) ! Cosine of solar zenith angle (column) for radiation [radians] ! Local parameters character(len=*), parameter :: module_name = '(orbital_data)' @@ -50,12 +51,18 @@ subroutine orbital_data_init(number_of_columns) call check_allocate(error_code, subroutine_name, & 'solar_zenith_angle(number_of_columns)', & file=__FILE__, line=__LINE__) + allocate(coszrs_rad(number_of_columns), source=FILL_R8, & + stat=error_code) + call check_allocate(error_code, subroutine_name, & + 'coszrs_rad(number_of_columns)', & + file=__FILE__, line=__LINE__) end subroutine orbital_data_init !======================================================================= - subroutine orbital_data_advance(calendar_day, latitudes, longitudes) + subroutine orbital_data_advance(calendar_day, latitudes, longitudes, use_rad_uniform_angle, & + rad_uniform_angle, dt_avg) !----------------------------------------------------------------------- ! @@ -69,6 +76,9 @@ subroutine orbital_data_advance(calendar_day, latitudes, longitudes) real(kind_phys), intent(in) :: calendar_day ! Fractional Julian calendar day (1.xx to 365.xx) real(kind_phys), intent(in) :: latitudes(:) ! Centered latitude (column) [radians] real(kind_phys), intent(in) :: longitudes(:) ! Centered longitude (column) [radians] + real(kind_phys), intent(in) :: rad_uniform_angle + real(kind_phys), intent(in) :: dt_avg + logical, intent(in) :: use_rad_uniform_angle integer :: i @@ -81,6 +91,20 @@ subroutine orbital_data_advance(calendar_day, latitudes, longitudes) solar_zenith_angle(i) = acos(shr_orb_cosz(calendar_day, latitudes(i), & longitudes(i), solar_declination)) end do + + ! Compute the cosine of solar zenith angle for radiation [radians] + if (use_rad_uniform_angle) then + do i = 1, size(latitudes) + coszrs_rad(i) = shr_orb_cosz(calendar_day, latitudes(i), longitudes(i), & + solar_declination, dt_avg, uniform_angle=rad_uniform_angle) + end do + else + do i = 1, size(latitudes) + ! if dt_avg /= 0, it triggers using avg coszrs + coszrs_rad(i) = shr_orb_cosz(calendar_day, latitudes(i), longitudes(i), & + solar_declination, dt_avg) + end do + end if end subroutine orbital_data_advance diff --git a/src/physics/utils/orbital_data.meta b/src/physics/utils/orbital_data.meta index bd134e5a..db74793e 100644 --- a/src/physics/utils/orbital_data.meta +++ b/src/physics/utils/orbital_data.meta @@ -22,3 +22,9 @@ type = real | kind = kind_phys dimensions = (horizontal_dimension) protected = True +[coszrs_rad] + standard_name = cosine_of_solar_zenith_angle_for_radiation + units = rad + type = real | kind = kind_phys + dimensions = (horizontal_dimension) + protected = True diff --git a/src/physics/utils/radiation_namelist.F90 b/src/physics/utils/radiation_namelist.F90 new file mode 100644 index 00000000..f5c75c78 --- /dev/null +++ b/src/physics/utils/radiation_namelist.F90 @@ -0,0 +1,133 @@ +module radiation_namelist +!-------------------------------------------------------------------------- +! +! This module provides namelist functionality for the radiation physics package(s). +! +!-------------------------------------------------------------------------- + + use ccpp_kinds, only: kind_phys + + implicit none + private + + ! public routines + public :: radiation_readnl + + ! public variables + !> \section arg_table_radiation_namelist Argument Table + !! \htmlinclude arg_table_radiation_namelist.html + !! + integer, public, protected :: iradsw = -1 ! freq. of shortwave radiation calc in time steps (positive) + ! or hours (negative). + integer, public, protected :: iradlw = -1 ! frequency of longwave rad. calc. in time steps (positive) + ! or hours (negative). + + integer, public, protected :: irad_always = 0 ! Specifies length of time in timesteps (positive) + ! or hours (negative) SW/LW radiation will be + ! run continuously from the start of an + ! initial or restart run + logical, public, protected :: use_rad_uniform_angle = .false. ! if true, use the namelist rad_uniform_angle for the coszrs calculation + real(kind_phys), public, protected :: rad_uniform_angle = -99._kind_phys ! radians + real(kind_phys), public, protected :: undefined_real_rad = -1._kind_phys + +!============================================================================== +contains +!============================================================================== + + subroutine radiation_readnl(nlfile) + + use shr_nl_mod, only: find_group_name => shr_nl_find_group_name + use mpi, only: mpi_logical, mpi_integer, mpi_real8 + use spmd_utils, only: mpicom + use cam_logfile, only: iulog + use cam_abortutils, only: endrun + use spmd_utils, only: masterproc + use time_manager, only: get_step_size + + !----------------------------------------------------------------------- + ! + ! Read CAM-SIMA's radiation namelist + ! + !----------------------------------------------------------------------- + + ! filepath for file containing namelist input + character(len=*), intent(in) :: nlfile + + ! Local variables + integer :: unitn, ierr + character(len=*), parameter :: subname = 'radiation_readnl' + character(len=shr_kind_cm) :: errmsg + integer :: dtime + + namelist /radiation_nl/ iradsw, iradlw, irad_always, use_rad_uniform_angle, & + rad_uniform_angle + + errmsg = '' + + if (masterproc) then + open(newunit=unitn, file=trim(nlfile), status='old') + call find_group_name(unitn, 'radiation_nl', status=ierr) + if (ierr == 0) then + read(unitn, radiation_nl, iostat=ierr, iomsg=errmsg) + if (ierr /= 0) then + call endrun(subname // ':: ERROR reading namelist:' // errmsg) + end if + end if + close(unitn) + end if + + ! Broadcast namelist variable + call mpi_bcast(use_rad_uniform_angle, 1, mpi_logical, masterproc, mpicom, ierr) + if (ierr /= 0) then + call endrun(sub//": FATAL: mpi_bcast: use_rad_uniform_angle", & + file=__FILE__, line=__LINE__) + end if + call mpi_bcast(iradsw, 1, mpi_integer, masterproc, mpicom, ierr) + if (ierr /= 0) then + call endrun(sub//": FATAL: mpi_bcast: iradsw", & + file=__FILE__, line=__LINE__) + end if + call mpi_bcast(iradlw, 1, mpi_integer, masterproc, mpicom, ierr) + if (ierr /= 0) then + call endrun(sub//": FATAL: mpi_bcast: iradlw", & + file=__FILE__, line=__LINE__) + end if + call mpi_bcast(irad_always, 1, mpi_integer, masterproc, mpicom, ierr) + if (ierr /= 0) then + call endrun(sub//": FATAL: mpi_bcast: irad_always", & + file=__FILE__, line=__LINE__) + end if + call mpi_bcast(rad_uniform_angle, 1, mpi_real8, masterprocid, mpicom, ierr) + if (ierr /= 0) then + call endrun(sub//": FATAL: mpi_bcast: rad_uniform_angle", & + file=__FILE__, line=__LINE__) + end if + + if (use_rad_uniform_angle .and. rad_uniform_angle == -99._r8) then + call endrun(sub//': ERROR - use_rad_uniform_angle is set to .true,' & + //' but rad_uniform_angle is not set ') + end if + + ! Convert iradsw, iradlw and irad_always from hours to timesteps if necessary + dtime = get_step_size() + if (iradsw < 0) iradsw = nint((-iradsw *3600._r8)/dtime) + if (iradlw < 0) iradlw = nint((-iradlw *3600._r8)/dtime) + if (irad_always < 0) irad_always = nint((-irad_always*3600._r8)/dtime) + + !----------------------------------------------------------------------- + ! Print runtime options to log. + !----------------------------------------------------------------------- + + if (masterproc) then + write(iulog,*) 'RRTMGP radiation scheme parameters:' + write(iulog,10) iradsw, iradlw, irad_always, use_rad_dt_cosz + end if + +10 format(' Frequency (timesteps) of Shortwave radiation calc: ',i5/, & + ' Frequency (timesteps) of Longwave Radiation calc: ',i5/, & + ' SW/LW calc done every timestep for first N steps. N=',i5/, & + ' Use average zenith angle: ',l5/) + + end subroutine radiation_readnl + +end module radiation_namelist diff --git a/src/physics/utils/radiation_namelist.meta b/src/physics/utils/radiation_namelist.meta new file mode 100644 index 00000000..8d68f9b2 --- /dev/null +++ b/src/physics/utils/radiation_namelist.meta @@ -0,0 +1,43 @@ +[ccpp-table-properties] + name = radiation_namelist + type = module + +[ccpp-arg-table] + name = radiation_namelist + type = module +[ iradsw ] + standard_name = frequency_of_shortwave_radiation_calculation + units = 1 + type = integer + dimensions = () + protected = True +[ iradlw ] + standard_name = frequency_of_longwave_radiation_calculation + units = 1 + type = integer + dimensions = () + protected = True +[ irad_always ] + standard_name = number_of_timesteps_to_force_radiation_calculation_after_initialization + units = count + type = integer + dimensions = () + protected = True +[ use_rad_uniform_angle ] + standard_name = use_radiation_uniform_angle_in_solar_zenith_angle_calculation + units = flag + type = logical + dimensions = () + protected = True +[ rad_uniform_angle ] + standard_name = radiation_uniform_angle_in_solar_zenith_angle_calculation + units = rad + type = real | kind = kind_phys + dimensions = () + protected = True +[ undefined_real_rad ] + standard_name = definition_of_undefined_for_radiation + units = 1 + type = real | kind = kind_phys + dimensions = () + protected = True From 1f19dd08316728373a16f1a570529866e6dd70a9 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Fri, 6 Jun 2025 12:09:18 -0600 Subject: [PATCH 002/150] latest atmos_phys tag --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 4198073d..f20d660b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = 61aea3a46db719b0dbfad31ecf47978f8df64e6b + fxtag = 04bb4ca3bbe4dcf44c302b9ee33570fb6e5d9117 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] From 8ac539c305ec56ff9a14c554c6c5fa69e41d11c1 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Fri, 6 Jun 2025 16:27:20 -0600 Subject: [PATCH 003/150] update atmos_phys submodule --- .gitmodules | 2 +- src/physics/ncar_ccpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index f20d660b..9c67f317 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = 04bb4ca3bbe4dcf44c302b9ee33570fb6e5d9117 + fxtag = b70cea45be1f3d4a9317443baecf41598bda64d1 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 61aea3a4..1a059472 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 61aea3a46db719b0dbfad31ecf47978f8df64e6b +Subproject commit 1a05947243b74b88b5cd669e233071d30ec1024b From bc961a0335d1bcb696e514de257a8b998e5c51a8 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Mon, 9 Jun 2025 10:03:40 -0600 Subject: [PATCH 004/150] cleanup; update atmos_phys hash --- .gitmodules | 2 +- src/control/runtime_obj.F90 | 1 - src/data/generate_registry_data.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9c67f317..fed1b04c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = b70cea45be1f3d4a9317443baecf41598bda64d1 + fxtag = a22d70ffa886510ed3e254942cf9e61c1ec91413 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/control/runtime_obj.F90 b/src/control/runtime_obj.F90 index 0879f5f3..03041f46 100644 --- a/src/control/runtime_obj.F90 +++ b/src/control/runtime_obj.F90 @@ -13,7 +13,6 @@ module runtime_obj character(len=*), public, parameter :: unset_str = 'UNSET' integer, public, parameter :: unset_int = huge(1) - !real(r8), public, parameter :: unset_real = huge(1.0_r8) ! Water vapor constituent standard name character(len=*), public, parameter :: wv_stdname = 'water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water' diff --git a/src/data/generate_registry_data.py b/src/data/generate_registry_data.py index 2e2c18fe..c2e1cf77 100755 --- a/src/data/generate_registry_data.py +++ b/src/data/generate_registry_data.py @@ -1067,7 +1067,6 @@ def write_definition(self, outfile, access, indent): def check_initial_values(self, physconst_vars,use_statements): """Raise an error if there are any initial values that are set to non-"used" and/or non-"physconst" variables""" - print(use_statements) for var in self.known_initial_value_vars: if var not in physconst_vars and not any(second == var for _, second in use_statements): emsg = f"Initial value '{var}' is not a physconst variable" From b9bdd6eaa3493a503388bf8ab553ddbce0d7f404 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Tue, 15 Jul 2025 10:21:40 -0600 Subject: [PATCH 005/150] update git-fleximod --- .lib/CODE_OF_CONDUCT.md | 107 +++ .lib/License | 20 + .lib/README.md | 108 +++ .lib/doc/Makefile | 20 + .lib/doc/conf.py | 26 + .lib/doc/index.rst | 24 + .lib/doc/make.bat | 35 + .lib/escomp_install | 25 + .lib/git-fleximod/git_fleximod/cli.py | 24 +- .../git-fleximod/git_fleximod/git_fleximod.py | 18 +- .../git-fleximod/git_fleximod/gitinterface.py | 29 +- .lib/git-fleximod/git_fleximod/submodule.py | 54 +- .lib/git-fleximod/pyproject.toml | 3 +- .lib/git-fleximod/tbump.toml | 2 +- .lib/git-fleximod/tests/conftest.py | 4 +- .lib/git_fleximod/__init__.py | 0 .lib/git_fleximod/cli.py | 151 ++++ .lib/git_fleximod/git_fleximod.py | 370 ++++++++++ .lib/git_fleximod/gitinterface.py | 115 +++ .lib/git_fleximod/gitmodules.py | 97 +++ .lib/git_fleximod/lstripreader.py | 43 ++ .lib/git_fleximod/metoflexi.py | 236 ++++++ .lib/git_fleximod/submodule.py | 428 +++++++++++ .lib/git_fleximod/utils.py | 365 +++++++++ .lib/poetry.lock | 693 ++++++++++++++++++ .lib/pyproject.toml | 42 ++ .lib/tbump.toml | 43 ++ .lib/tests/__init__.py | 3 + .lib/tests/conftest.py | 150 ++++ .lib/tests/test_a_import.py | 8 + .lib/tests/test_b_update.py | 26 + .lib/tests/test_c_required.py | 30 + .lib/tests/test_d_complex.py | 66 ++ .lib/tests/test_e_complex_update.py | 69 ++ src/physics/ncar_ccpp | 2 +- 35 files changed, 3406 insertions(+), 30 deletions(-) create mode 100644 .lib/CODE_OF_CONDUCT.md create mode 100644 .lib/License create mode 100644 .lib/README.md create mode 100644 .lib/doc/Makefile create mode 100644 .lib/doc/conf.py create mode 100644 .lib/doc/index.rst create mode 100644 .lib/doc/make.bat create mode 100644 .lib/escomp_install create mode 100644 .lib/git_fleximod/__init__.py create mode 100644 .lib/git_fleximod/cli.py create mode 100755 .lib/git_fleximod/git_fleximod.py create mode 100644 .lib/git_fleximod/gitinterface.py create mode 100644 .lib/git_fleximod/gitmodules.py create mode 100644 .lib/git_fleximod/lstripreader.py create mode 100755 .lib/git_fleximod/metoflexi.py create mode 100644 .lib/git_fleximod/submodule.py create mode 100644 .lib/git_fleximod/utils.py create mode 100644 .lib/poetry.lock create mode 100644 .lib/pyproject.toml create mode 100644 .lib/tbump.toml create mode 100644 .lib/tests/__init__.py create mode 100644 .lib/tests/conftest.py create mode 100644 .lib/tests/test_a_import.py create mode 100644 .lib/tests/test_b_update.py create mode 100644 .lib/tests/test_c_required.py create mode 100644 .lib/tests/test_d_complex.py create mode 100644 .lib/tests/test_e_complex_update.py diff --git a/.lib/CODE_OF_CONDUCT.md b/.lib/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..84f2925b --- /dev/null +++ b/.lib/CODE_OF_CONDUCT.md @@ -0,0 +1,107 @@ +# Contributor Code of Conduct +_The Contributor Code of Conduct is for participants in our software projects and community._ + +## Our Pledge +We, as contributors, creators, stewards, and maintainers (participants), of **git-fleximod** pledge to make participation in +our software, system or hardware project and community a safe, productive, welcoming and inclusive experience for everyone. +All participants are required to abide by this Code of Conduct. +This includes respectful treatment of everyone regardless of age, body size, disability, ethnicity, gender identity or expression, +level of experience, nationality, political affiliation, veteran status, pregnancy, genetic information, physical appearance, race, +religion, or sexual orientation, as well as any other characteristic protected under applicable US federal or state law. + +## Our Standards +Examples of behaviors that contribute to a positive environment include: + +* All participants are treated with respect and consideration, valuing a diversity of views and opinions +* Be considerate, respectful, and collaborative +* Communicate openly with respect for others, critiquing ideas rather than individuals and gracefully accepting criticism +* Acknowledging the contributions of others +* Avoid personal attacks directed toward other participants +* Be mindful of your surroundings and of your fellow participants +* Alert UCAR staff and suppliers/vendors if you notice a dangerous situation or someone in distress +* Respect the rules and policies of the project and venue + +Examples of unacceptable behavior include, but are not limited to: + +* Harassment, intimidation, or discrimination in any form +* Physical, verbal, or written abuse by anyone to anyone, including repeated use of pronouns other than those requested +* Unwelcome sexual attention or advances +* Personal attacks directed at other guests, members, participants, etc. +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Alarming, intimidating, threatening, or hostile comments or conduct +* Inappropriate use of nudity and/or sexual images +* Threatening or stalking anyone, including a participant +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Scope +This Code of Conduct applies to all spaces managed by the Project whether they be physical, online or face-to-face. +This includes project code, code repository, associated web pages, documentation, mailing lists, project websites and wiki pages, +issue tracker, meetings, telecons, events, project social media accounts, and any other forums created by the project team which the +community uses for communication. +In addition, violations of this Code of Conduct outside these spaces may affect a person's ability to participate within them. +Representation of a project may be further defined and clarified by project maintainers. + +## Community Responsibilities +Everyone in the community is empowered to respond to people who are showing unacceptable behavior. +They can talk to them privately or publicly. +Anyone requested to stop unacceptable behavior is expected to comply immediately. +If the behavior continues concerns may be brought to the project administrators or to any other party listed in the +[Reporting](#reporting) section below. + +## Project Administrator Responsibilities +Project administrators are responsible for clarifying the standards of acceptable behavior and are encouraged to model appropriate +behavior and provide support when people in the community point out inappropriate behavior. +Project administrator(s) are normally the ones that would be tasked to carry out the actions in the [Consequences](#consequences) +section below. + +Project administrators are also expected to keep this Code of Conduct updated with the main one housed at UCAR, as listed below in +the [Attribution](#attribution) section. + +## Reporting +Instances of unacceptable behavior can be brought to the attention of the project administrator(s) who may take any action as +outlined in the [Consequences](#consequences) section below. +However, making a report to a project administrator is not considered an 'official report' to UCAR. + +Instances of unacceptable behavior may also be reported directly to UCAR pursuant to [UCAR's Harassment Reporting and Complaint +Procedure](https://www2.fin.ucar.edu/procedures/hr/harassment-reporting-and-complaint-procedure), or anonymously through [UCAR's +EthicsPoint Hotline](https://www2.fin.ucar.edu/ethics/anonymous-reporting). + +Complaints received by UCAR will be handled pursuant to the procedures outlined in UCAR's Harassment Reporting and Complaint +Procedure. +Complaints to UCAR will be held as confidential as practicable under the circumstances, and retaliation against a person who +initiates a complaint or an inquiry about inappropriate behavior will not be tolerated. + +Any Contributor can use these reporting methods even if they are not directly affiliated with UCAR. +The Frequently Asked Questions (FAQ) page for reporting is [here](https://www2.fin.ucar.edu/procedures/hr/reporting-faqs). + +## Consequences +Upon receipt of a complaint, the project administrator(s) may take any action deemed necessary and appropriate under the +circumstances. +Such action can include things such as: removing, editing, or rejecting comments, commits, code, wiki edits, email, issues, and +other contributions that are not aligned to this Code of Conduct, or banning temporarily or permanently any contributor for other +behaviors that are deemed inappropriate, threatening, offensive, or harmful. +Project administrators also have the right to report violations to UCAR HR and/or UCAR's Office of Diversity, Equity and Inclusion +(ODEI), as well as a participant's home institution and/or law enforcement. +In the event an incident is reported to UCAR, UCAR will follow its Harassment Reporting and Complaint Procedure. + +## Process for Changes +All UCAR managed projects are required to adopt this Contributor Code of Conduct. +Adoption is assumed even if not expressly stated in the repository. +Projects should fill in sections where prompted with project-specific information, including, project name and adoption date. + +Projects that adopt this Code of Conduct need to stay up to date with UCAR's Contributor Code of Conduct, linked with a DOI in the +[Attribution](#attribution) section below. +Projects can make limited substantive changes to the Code of Conduct, however, the changes must be limited in scope and may not +contradict the UCAR Contributor Code of Conduct. + +## Attribution +This Code of Conduct was originally adapted from the [Contributor Covenant](http://contributor-covenant.org/version/1/4), version +1.4. +We then aligned it with the UCAR Participant Code of Conduct, which also borrows from the American Geophysical Union (AGU) Code of +Conduct. +The UCAR Participant Code of Conduct applies to both UCAR employees as well as participants in activities run by UCAR. +The original version of this for all software projects that have strong management from UCAR or UCAR staff is available on the UCAR +website at https://doi.org/10.5065/6w2c-a132. +The date that it was adopted by this project was **Feb/13/2018**. +When responding to complaints, UCAR HR and ODEI will do so based on the latest published version. +Therefore, any project-specific changes should follow the [Process for Changes](#process-for-changes) section above. diff --git a/.lib/License b/.lib/License new file mode 100644 index 00000000..88bc2251 --- /dev/null +++ b/.lib/License @@ -0,0 +1,20 @@ +Copyright 2024 NSF National Center for Atmospheric Sciences (NCAR) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +“Software”), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/.lib/README.md b/.lib/README.md new file mode 100644 index 00000000..53917da4 --- /dev/null +++ b/.lib/README.md @@ -0,0 +1,108 @@ +# git-fleximod + +Flexible, Enhanced Submodule Management for Git + +## Overview + +Git-fleximod is a Python-based tool that extends Git's submodule and sparse checkout capabilities, offering additional features for managing submodules in a more flexible and efficient way. + +## Installation + + If you choose to locate git-fleximod in your path you can access it via command: git fleximod + +## Usage + + Basic Usage: + git fleximod [options] + Available Commands: + status: Display the status of submodules. + update: Update submodules to the tag indicated in .gitmodules variable fxtag. + test: Make sure that fxtags and submodule hashes are consistant, + make sure that official urls (as defined by fxDONOTUSEurl) are set + make sure that fxtags are defined for all submodules + Additional Options: + See git fleximod --help for more details. + +## Supported .gitmodules Variables + + fxtag: Specify a specific tag or branch to checkout for a submodule. + fxrequired: Mark a submodule's checkout behavior, with allowed values: + - ToplevelRequired: Top-level and required (checked out only when this is the Toplevel module). + - ToplevelOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module). + - AlwaysRequired: Always required (always checked out). + - AlwaysOptional: Always optional (checked out with --optional flag). + fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths. + fxDONOTUSEurl: This is the url used in the test subcommand to assure that protected branches do not point to forks + **NOTE** the fxDONOTUSEurl variable is only used to identify the official project repository and should not be + changed by users. Use the url variable to change to a fork if desired. + +## Sparse Checkouts + + To enable sparse checkout for a submodule, set the fxsparse variable + in the .gitmodules file to the path of a file containing the desired + sparse checkout paths. Git-fleximod will automatically configure + sparse checkout based on this file when applicable commands are run. + See [git-sparse-checkout](https://git-scm.com/docs/git-sparse-checkout#_internalsfull_pattern_set) + for details on the format of this file. + +## Tests + + The git fleximod test action is designed to be used by, for example, github workflows + to assure that protected branches are consistant with respect to submodule hashes and fleximod fxtags + +## Examples + +Here are some common usage examples: + +Update all submodules, including optional ones: +```bash + git fleximod update --optional +``` + +Updating a specific submodule to the fxtag indicated in .gitmodules: + +```bash + git fleximod update submodule-name +``` +Example .gitmodules entry: +```ini, toml + [submodule "cosp2"] + path = src/physics/cosp2/src + url = https://github.com/CFMIP/COSPv2.0 + fxsparse = ../.cosp_sparse_checkout + fxrequired = AlwaysRequired + fxtag = v2.1.4cesm +``` +Explanation: + +This entry indicates that the submodule named cosp2 at tag v2.1.4cesm +should be checked out into the directory src/physics/cosp2/src +relative to the .gitmodules directory. It should be checked out from +the URL https://github.com/CFMIP/COSPv2.0 and use sparse checkout as +described in the file ../.cosp_sparse_checkout relative to the path +directory. It should be checked out anytime this .gitmodules entry is +read. + +Additional example: +```ini, toml + [submodule "cime"] + path = cime + url = https://github.com/jedwards4b/cime + fxrequired = ToplevelRequired + fxtag = cime6.0.198_rme01 +``` + +Explanation: + +This entry indicates that the submodule cime should be checked out +into a directory named cime at tag cime6.0.198_rme01 from the URL +https://github.com/jedwards4b/cime. This should only be done if +the .gitmodules file is at the top level of the repository clone. + +## Contributing + +We welcome contributions! Please see the CONTRIBUTING.md file for guidelines. + +## License + +Git-fleximod is released under the MIT License. diff --git a/.lib/doc/Makefile b/.lib/doc/Makefile new file mode 100644 index 00000000..d4bb2cbb --- /dev/null +++ b/.lib/doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/.lib/doc/conf.py b/.lib/doc/conf.py new file mode 100644 index 00000000..423099ee --- /dev/null +++ b/.lib/doc/conf.py @@ -0,0 +1,26 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "git-fleximod" +author = "Jim Edwards " +release = "0.4.0" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ["sphinx_argparse_cli"] + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "alabaster" +html_static_path = ["_static"] diff --git a/.lib/doc/index.rst b/.lib/doc/index.rst new file mode 100644 index 00000000..0f9c1a7f --- /dev/null +++ b/.lib/doc/index.rst @@ -0,0 +1,24 @@ +.. git-fleximod documentation master file, created by + sphinx-quickstart on Sat Feb 3 12:02:22 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to git-fleximod's documentation! +======================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: +.. module:: sphinxcontrib.autoprogram +.. sphinx_argparse_cli:: + :module: git_fleximod.cli + :func: get_parser + :prog: git-fleximod + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/.lib/doc/make.bat b/.lib/doc/make.bat new file mode 100644 index 00000000..32bb2452 --- /dev/null +++ b/.lib/doc/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/.lib/escomp_install b/.lib/escomp_install new file mode 100644 index 00000000..ae782e72 --- /dev/null +++ b/.lib/escomp_install @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# updates git-fleximod in an ESCOMP model +# this script should be run from the model root directory, it expects +# git-fleximod to already be installed with the script in bin +# and the classes in lib/python/site-packages +import sys +import shutil +import os + +from glob import iglob + +fleximod_root = sys.argv[1] +fleximod_path = os.path.join(fleximod_root,"src","git-fleximod") +if os.path.isfile(fleximod_path): + with open(fleximod_path,"r") as f: + fleximod = f.readlines() + with open(os.path.join(".","bin","git-fleximod"),"w") as f: + for line in fleximod: + f.write(line) + if "import argparse" in line: + f.write('\nsys.path.append(os.path.join(os.path.dirname(__file__),"..","lib","python","site-packages"))\n\n') + + for file in iglob(os.path.join(fleximod_root, "src", "fleximod", "*.py")): + shutil.copy(file, + os.path.join("lib","python","site-packages","fleximod",os.path.basename(file))) diff --git a/.lib/git-fleximod/git_fleximod/cli.py b/.lib/git-fleximod/git_fleximod/cli.py index 131466b9..9169b338 100644 --- a/.lib/git-fleximod/git_fleximod/cli.py +++ b/.lib/git-fleximod/git_fleximod/cli.py @@ -1,8 +1,26 @@ from pathlib import Path -import argparse +import argparse, os, sys +from importlib.resources import files from git_fleximod import utils -__version__ = "0.9.4" +__version__ = "1.0.0" + +class CustomArgumentParser(argparse.ArgumentParser): + def print_help(self, file=None): + # First print the default help message + super().print_help(file) + + # Then append the contents of README.md + candidate_paths = [ + os.path.join(sys.prefix, "share", "your-package", "README.md"), + os.path.join(os.path.dirname(__file__), "..", "README.md") # fallback for dev + ] + for path in candidate_paths: + if os.path.exists(path): + with open(path) as f: + print( f.read(), file=file) + return + print( "README.md not found.", file=file) def find_root_dir(filename=".gitmodules"): """ finds the highest directory in tree @@ -32,7 +50,7 @@ def get_parser(): description = """ %(prog)s manages checking out groups of gitsubmodules with additional support for Earth System Models """ - parser = argparse.ArgumentParser( + parser = CustomArgumentParser( description=description, formatter_class=argparse.RawDescriptionHelpFormatter ) diff --git a/.lib/git-fleximod/git_fleximod/git_fleximod.py b/.lib/git-fleximod/git_fleximod/git_fleximod.py index 2c2601fa..b3c4fece 100755 --- a/.lib/git-fleximod/git_fleximod/git_fleximod.py +++ b/.lib/git-fleximod/git_fleximod/git_fleximod.py @@ -9,6 +9,7 @@ import shutil import logging import textwrap +import asyncio from git_fleximod import utils from git_fleximod import cli from git_fleximod.gitinterface import GitInterface @@ -218,10 +219,10 @@ def git_toplevelroot(root_dir, logger): _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") return superroot -def submodules_update(gitmodules, root_dir, requiredlist, force): - for name in gitmodules.sections(): +async def submodules_update(gitmodules, root_dir, requiredlist, force): + async def update_submodule(name, requiredlist, force): submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) - + _, needsupdate, localmods, testfails = submod.status() if not submod.fxrequired: submod.fxrequired = "AlwaysRequired" @@ -239,11 +240,11 @@ def submodules_update(gitmodules, root_dir, requiredlist, force): if "Optional" in fxrequired and "Optional" not in requiredlist: if fxrequired.startswith("Always"): print(f"Skipping optional component {name:>20}") - continue + return # continue to next submodule optional = "AlwaysOptional" in requiredlist if fxrequired in requiredlist: - submod.update() + await submod.update() repodir = os.path.join(root_dir, submod.path) if os.path.exists(os.path.join(repodir, ".gitmodules")): # recursively handle this checkout @@ -252,7 +253,10 @@ def submodules_update(gitmodules, root_dir, requiredlist, force): newrequiredlist = ["AlwaysRequired"] if optional: newrequiredlist.append("AlwaysOptional") - submodules_update(gitsubmodules, repodir, newrequiredlist, force=force) + await submodules_update(gitsubmodules, repodir, newrequiredlist, force=force) + + tasks = [update_submodule(name, requiredlist, force) for name in gitmodules.sections()] + await asyncio.gather(*tasks) def local_mods_output(): text = """\ @@ -346,7 +350,7 @@ def main(): sys.exit(f"No submodule components found, root_dir={root_dir}") retval = 0 if action == "update": - submodules_update(gitmodules, root_dir, fxrequired, force) + asyncio.run(submodules_update(gitmodules, root_dir, fxrequired, force)) elif action == "status": tfails, lmods, updates = submodules_status(gitmodules, root_dir, toplevel=True) if tfails + lmods + updates > 0: diff --git a/.lib/git-fleximod/git_fleximod/gitinterface.py b/.lib/git-fleximod/git_fleximod/gitinterface.py index fb20883c..1a736e4e 100644 --- a/.lib/git-fleximod/git_fleximod/gitinterface.py +++ b/.lib/git-fleximod/git_fleximod/gitinterface.py @@ -2,6 +2,7 @@ import sys from . import utils from pathlib import Path +import asyncio class GitInterface: def __init__(self, repo_path, logger): @@ -47,8 +48,7 @@ def _init_git_repo(self): command = ("git", "-C", str(self.repo_path), "init") utils.execute_subprocess(command) - # pylint: disable=unused-argument - def git_operation(self, operation, *args, **kwargs): + def _git_operation_command(self, operation, args): newargs = [] for a in args: # Do not use ssh interface @@ -56,7 +56,11 @@ def git_operation(self, operation, *args, **kwargs): a = a.replace("git@github.com:", "https://github.com/") newargs.append(a) - command = self._git_command(operation, *newargs) + return self._git_command(operation, *newargs) + + # pylint: disable=unused-argument + def git_operation(self, operation, *args, **kwargs): + command = self._git_operation_command(operation, args) if isinstance(command, list): try: status, output = utils.execute_subprocess(command, status_to_caller=True, output_to_caller=True) @@ -66,6 +70,25 @@ def git_operation(self, operation, *args, **kwargs): else: return 0, command + # pylint: disable=unused-argument + async def git_operation_async(self, operation, *args, **kwargs): + command = self._git_operation_command(operation, args) + if isinstance(command, list): + try: + process = await asyncio.create_subprocess_exec( + *command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await process.communicate() + status = process.returncode + output = stdout.decode().strip() if stdout else stderr.decode().strip() + return status, output + except Exception as e: + sys.exit(e) + else: + return 0, command + def config_get_value(self, section, name): if self._use_module: config = self.repo.config_reader() diff --git a/.lib/git-fleximod/git_fleximod/submodule.py b/.lib/git-fleximod/git_fleximod/submodule.py index 75d9dd4e..cb9abe50 100644 --- a/.lib/git-fleximod/git_fleximod/submodule.py +++ b/.lib/git-fleximod/git_fleximod/submodule.py @@ -284,17 +284,18 @@ def sparse_checkout(self): if not os.path.isdir(infodir): os.makedirs(infodir) gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout")) - if os.path.isfile(gitsparse): - self.logger.warning( - "submodule {} is already initialized {}".format(self.name, rootdotgit) - ) - return - - with utils.pushd(sprep_repo): + if os.path.isfile(gitsparse): + self.logger.warning( + "submodule {} is already initialized {}".format(self.name, rootdotgit) + ) + os.remove(gitsparse) + if os.path.isfile(self.fxsparse): - shutil.copy(self.fxsparse, gitsparse) - + else: + self.logger.warning( + "submodule {} could not find {}".format(self.name, self.fxsparse) + ) # Finally checkout the repo sprepo_git.git_operation("fetch", "origin", "--tags") @@ -303,11 +304,21 @@ def sparse_checkout(self): print(f"Error checking out {self.name:>20} at {self.fxtag}") else: print(f"Successfully checked out {self.name:>20} at {self.fxtag}") + status,f = sprepo_git.git_operation("status") + checkout_nextline = False + # Restore any files deleted from sandbox + for line in f.split(): + if checkout_nextline: + status, _ = sprepo_git.git_operation("checkout", line) + checkout_nextline = False + if line == "deleted:": + checkout_nextline = True + rgit.config_set_value('submodule.' + self.name, "active", "true") rgit.config_set_value('submodule.' + self.name, "url", self.url) rgit.config_set_value('submodule.' + self.name, "path", self.path) - def update(self): + async def update(self): """ Updates the submodule to the latest or specified version. @@ -340,7 +351,10 @@ def update(self): repo_exists = True # Look for a .gitmodules file in the newly checkedout repo if self.fxsparse: - print(f"Sparse checkout {self.name} fxsparse {self.fxsparse}") + print(f"Sparse checkout {self.name} fxsparse {self.fxsparse} {git.repo_path}") + if not os.path.isfile(self.fxsparse): + self.logger.info("Submodule {} fxsparse file not found") + self.sparse_checkout() else: if not repo_exists and self.url: @@ -378,7 +392,8 @@ def update(self): git.git_operation("submodule", "add", "--name", self.name, "--", self.url, self.path) if not repo_exists: - git.git_operation("submodule", "update", "--init", "--", self.path) + git.git_operation("submodule", "init", "--", self.path) + await git.git_operation_async("submodule", "update", "--", self.path) if self.fxtag: smgit = GitInterface(repodir, self.logger) @@ -408,6 +423,17 @@ def update(self): if fxtag and fxtag not in tags: git.git_operation("fetch", newremote, "--tags") status, atag = git.git_operation("describe", "--tags", "--always") + status, files = git.git_operation("diff", "--name-only") + modfiles = [] + moddirs = [] + if files: + for f in files.split(): + if not os.path.exists(f): + git.git_operation("checkout",f) + elif os.path.isdir(f): + moddirs.append(f) + else: + modfiles.append(f) if fxtag and fxtag != atag: try: status, _ = git.git_operation("checkout", fxtag) @@ -419,6 +445,10 @@ def update(self): elif not fxtag: print(f"No fxtag found for submodule {self.name:>20}") + elif modfiles: + print(f"{self.name:>20} has modified files: {modfiles}") + elif moddirs: + print(f"{self.name:>20} has modified directories: {moddirs}") else: print(f"{self.name:>20} up to date.") diff --git a/.lib/git-fleximod/pyproject.toml b/.lib/git-fleximod/pyproject.toml index 65924ff9..76ef0ef8 100644 --- a/.lib/git-fleximod/pyproject.toml +++ b/.lib/git-fleximod/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "git-fleximod" -version = "0.9.4" +version = "1.0.0" description = "Extended support for git-submodule and git-sparse-checkout" authors = ["Jim Edwards "] maintainers = ["Jim Edwards "] @@ -11,6 +11,7 @@ keywords = ["git", "submodule", "sparse-checkout"] packages = [ { include = "git_fleximod"}, { include = "doc"}, +{ include = "README.md"}, ] [tool.poetry.scripts] diff --git a/.lib/git-fleximod/tbump.toml b/.lib/git-fleximod/tbump.toml index be0b799d..4ac513dd 100644 --- a/.lib/git-fleximod/tbump.toml +++ b/.lib/git-fleximod/tbump.toml @@ -2,7 +2,7 @@ github_url = "https://github.com/jedwards4b/git-fleximod/" [version] -current = "0.9.4" +current = "1.0.0" # Example of a semver regexp. # Make sure this matches current_version before diff --git a/.lib/git-fleximod/tests/conftest.py b/.lib/git-fleximod/tests/conftest.py index 1dd1b86f..44d28e17 100644 --- a/.lib/git-fleximod/tests/conftest.py +++ b/.lib/git-fleximod/tests/conftest.py @@ -32,7 +32,7 @@ def logger(): "submodule_name": "test_optional", "status1" : "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", "status2" : "test_optional at tag MPIserial_2.4.0", - "status3" : "test_optional not checked out, out of sync at tag MPIserial_2.5.1, expected tag is MPIserial_2.4.0 (optional)", + "status3" : "test_optional not checked out, out of sync at tag MPIserial_2.5.4, expected tag is MPIserial_2.4.0 (optional)", "status4" : "test_optional at tag MPIserial_2.4.0", "gitmodules_content": """ [submodule "test_optional"] @@ -46,7 +46,7 @@ def logger(): "submodule_name": "test_alwaysoptional", "status1" : "test_alwaysoptional MPIserial_2.3.0 is out of sync with .gitmodules e5cf35c", "status2" : "test_alwaysoptional at hash e5cf35c", - "status3" : "out of sync at tag MPIserial_2.5.1, expected tag is e5cf35c", + "status3" : "out of sync at tag MPIserial_2.5.4, expected tag is e5cf35c", "status4" : "test_alwaysoptional at hash e5cf35c", "gitmodules_content": """ [submodule "test_alwaysoptional"] diff --git a/.lib/git_fleximod/__init__.py b/.lib/git_fleximod/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/.lib/git_fleximod/cli.py b/.lib/git_fleximod/cli.py new file mode 100644 index 00000000..9169b338 --- /dev/null +++ b/.lib/git_fleximod/cli.py @@ -0,0 +1,151 @@ +from pathlib import Path +import argparse, os, sys +from importlib.resources import files +from git_fleximod import utils + +__version__ = "1.0.0" + +class CustomArgumentParser(argparse.ArgumentParser): + def print_help(self, file=None): + # First print the default help message + super().print_help(file) + + # Then append the contents of README.md + candidate_paths = [ + os.path.join(sys.prefix, "share", "your-package", "README.md"), + os.path.join(os.path.dirname(__file__), "..", "README.md") # fallback for dev + ] + for path in candidate_paths: + if os.path.exists(path): + with open(path) as f: + print( f.read(), file=file) + return + print( "README.md not found.", file=file) + +def find_root_dir(filename=".gitmodules"): + """ finds the highest directory in tree + which contains a file called filename """ + try: + root = utils.execute_subprocess(["git","rev-parse", "--show-toplevel"], + output_to_caller=True ).rstrip() + except: + d = Path.cwd() + root = Path(d.root) + dirlist = [] + dl = d + while dl != root: + dirlist.append(dl) + dl = dl.parent + dirlist.append(root) + dirlist.reverse() + + for dl in dirlist: + attempt = dl / filename + if attempt.is_file(): + return str(dl) + return None + return Path(root) + +def get_parser(): + description = """ + %(prog)s manages checking out groups of gitsubmodules with additional support for Earth System Models + """ + parser = CustomArgumentParser( + description=description, formatter_class=argparse.RawDescriptionHelpFormatter + ) + + # + # user options + # + choices = ["update", "status", "test"] + parser.add_argument( + "action", + choices=choices, + default="update", + help=f"Subcommand of git-fleximod, choices are {choices[:-1]}", + ) + + parser.add_argument( + "components", + nargs="*", + help="Specific component(s) to checkout. By default, " + "all required submodules are checked out.", + ) + + parser.add_argument( + "-C", + "--path", + default=find_root_dir(), + help="Toplevel repository directory. Defaults to top git directory relative to current.", + ) + + parser.add_argument( + "-g", + "--gitmodules", + nargs="?", + default=".gitmodules", + help="The submodule description filename. " "Default: %(default)s.", + ) + + parser.add_argument( + "-x", + "--exclude", + nargs="*", + help="Component(s) listed in the gitmodules file which should be ignored.", + ) + parser.add_argument( + "-f", + "--force", + action="store_true", + default=False, + help="Override cautions and update or checkout over locally modified repository.", + ) + + parser.add_argument( + "-o", + "--optional", + action="store_true", + default=False, + help="By default only the required submodules " + "are checked out. This flag will also checkout the " + "optional submodules relative to the toplevel directory.", + ) + + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Output additional information to " + "the screen and log file. This flag can be " + "used up to two times, increasing the " + "verbosity level each time.", + ) + + parser.add_argument( + "-V", + "--version", + action="version", + version=f"%(prog)s {__version__}", + help="Print version and exit.", + ) + + # + # developer options + # + parser.add_argument( + "--backtrace", + action="store_true", + help="DEVELOPER: show exception backtraces as extra " "debugging output", + ) + + parser.add_argument( + "-d", + "--debug", + action="store_true", + default=False, + help="DEVELOPER: output additional debugging " + "information to the screen and log file.", + ) + + return parser diff --git a/.lib/git_fleximod/git_fleximod.py b/.lib/git_fleximod/git_fleximod.py new file mode 100755 index 00000000..b3c4fece --- /dev/null +++ b/.lib/git_fleximod/git_fleximod.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python +import sys + +MIN_PYTHON = (3, 7) +if sys.version_info < MIN_PYTHON: + sys.exit("Python %s.%s or later is required." % MIN_PYTHON) + +import os +import shutil +import logging +import textwrap +import asyncio +from git_fleximod import utils +from git_fleximod import cli +from git_fleximod.gitinterface import GitInterface +from git_fleximod.gitmodules import GitModules +from git_fleximod.submodule import Submodule + +# logger variable is global +logger = None + + +def fxrequired_allowed_values(): + return ["ToplevelRequired", "ToplevelOptional", "AlwaysRequired", "AlwaysOptional", "TopLevelRequired", "TopLevelOptional"] + + +def commandline_arguments(args=None): + parser = cli.get_parser() + + if args: + options = parser.parse_args(args) + else: + options = parser.parse_args() + + # explicitly listing a component overrides the optional flag + if options.optional or options.components: + fxrequired = fxrequired_allowed_values() + else: + fxrequired = ["ToplevelRequired", "AlwaysRequired", "TopLevelRequired"] + + action = options.action + if not action: + action = "update" + handlers = [logging.StreamHandler()] + + if options.debug: + try: + open("fleximod.log", "w") + except PermissionError: + sys.exit("ABORT: Could not write file fleximod.log") + level = logging.DEBUG + handlers.append(logging.FileHandler("fleximod.log")) + elif options.verbose: + level = logging.INFO + else: + level = logging.WARNING + # Configure the root logger + logging.basicConfig( + level=level, format="%(name)s - %(levelname)s - %(message)s", handlers=handlers + ) + + if hasattr(options, "version"): + exit() + + return ( + options.path, + options.gitmodules, + fxrequired, + options.components, + options.exclude, + options.force, + action, + ) + + +def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master"): + """ + This function performs a sparse checkout of a git submodule. It does so by first creating the .git/info/sparse-checkout fileq + in the submodule and then checking out the desired tag. If the submodule is already checked out, it will not be checked out again. + Creating the sparse-checkout file first prevents the entire submodule from being checked out and then removed. This is important + because the submodule may have a large number of files and checking out the entire submodule and then removing it would be time + and disk space consuming. + + Parameters: + root_dir (str): The root directory for the git operation. + name (str): The name of the submodule. + url (str): The URL of the submodule. + path (str): The path to the submodule. + sparsefile (str): The sparse file for the submodule. + tag (str, optional): The tag to checkout. Defaults to "master". + + Returns: + None + """ + logger.info("Called sparse_checkout for {}".format(name)) + rgit = GitInterface(root_dir, logger) + superroot = git_toplevelroot(root_dir, logger) + + if superroot: + gitroot = superroot.strip() + else: + gitroot = root_dir.strip() + assert os.path.isdir(os.path.join(gitroot, ".git")) + # first create the module directory + if not os.path.isdir(os.path.join(root_dir, path)): + os.makedirs(os.path.join(root_dir, path)) + + # initialize a new git repo and set the sparse checkout flag + sprep_repo = os.path.join(root_dir, path) + sprepo_git = GitInterface(sprep_repo, logger) + if os.path.exists(os.path.join(sprep_repo, ".git")): + try: + logger.info("Submodule {} found".format(name)) + chk = sprepo_git.config_get_value("core", "sparseCheckout") + if chk == "true": + logger.info("Sparse submodule {} already checked out".format(name)) + return + except NoOptionError: + logger.debug("Sparse submodule {} not present".format(name)) + except Exception as e: + utils.fatal_error("Unexpected error {} occured.".format(e)) + + sprepo_git.config_set_value("core", "sparseCheckout", "true") + + # set the repository remote + + logger.info("Setting remote origin in {}/{}".format(root_dir, path)) + _, remotelist = sprepo_git.git_operation("remote", "-v") + if url not in remotelist: + sprepo_git.git_operation("remote", "add", "origin", url) + + topgit = os.path.join(gitroot, ".git") + + if gitroot != root_dir and os.path.isfile(os.path.join(root_dir, ".git")): + with open(os.path.join(root_dir, ".git")) as f: + gitpath = os.path.relpath( + os.path.join(root_dir, f.read().split()[1]), + start=os.path.join(root_dir, path), + ) + topgit = os.path.join(gitpath, "modules") + else: + topgit = os.path.relpath( + os.path.join(root_dir, ".git", "modules"), + start=os.path.join(root_dir, path), + ) + + with utils.pushd(sprep_repo): + if not os.path.isdir(topgit): + os.makedirs(topgit) + topgit += os.sep + name + + if os.path.isdir(os.path.join(root_dir, path, ".git")): + with utils.pushd(sprep_repo): + if os.path.isdir(os.path.join(topgit,".git")): + shutil.rmtree(os.path.join(topgit,".git")) + shutil.move(".git", topgit) + with open(".git", "w") as f: + f.write("gitdir: " + os.path.relpath(topgit)) + # assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo))) + gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout")) + if os.path.isfile(gitsparse): + logger.warning( + "submodule {} is already initialized {}".format(name, topgit) + ) + return + + with utils.pushd(sprep_repo): + if os.path.isfile(sparsefile): + shutil.copy(sparsefile, gitsparse) + + + # Finally checkout the repo + sprepo_git.git_operation("fetch", "origin", "--tags") + sprepo_git.git_operation("checkout", tag) + + print(f"Successfully checked out {name:>20} at {tag}") + rgit.config_set_value(f'submodule "{name}"', "active", "true") + rgit.config_set_value(f'submodule "{name}"', "url", url) + +def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger): + path = gitmodules.get(name, "path") + url = gitmodules.get(name, "url") + assert path and url, f"Malformed .gitmodules file {path} {url}" + tag = gitmodules.get(name, "fxtag") + if not tag: + tag = gitmodules.get(name, "hash") + fxurl = gitmodules.get(name, "fxDONOTUSEurl") + fxsparse = gitmodules.get(name, "fxsparse") + fxrequired = gitmodules.get(name, "fxrequired") + return Submodule(root_dir, name, path, url, fxtag=tag, fxurl=fxurl, fxsparse=fxsparse, fxrequired=fxrequired, logger=logger) + +def submodules_status(gitmodules, root_dir, toplevel=False, depth=0): + testfails = 0 + localmods = 0 + needsupdate = 0 + wrapper = textwrap.TextWrapper(initial_indent=' '*(depth*10), width=120,subsequent_indent=' '*(depth*20)) + for name in gitmodules.sections(): + submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) + + result,n,l,t = submod.status() + if toplevel or not submod.toplevel(): + print(wrapper.fill(result)) + testfails += t + localmods += l + needsupdate += n + subdir = os.path.join(root_dir, submod.path) + if os.path.exists(os.path.join(subdir, ".gitmodules")): + gsubmod = GitModules(logger, confpath=subdir) + t,l,n = submodules_status(gsubmod, subdir, depth=depth+1) + if toplevel or not submod.toplevel(): + testfails += t + localmods += l + needsupdate += n + + return testfails, localmods, needsupdate + +def git_toplevelroot(root_dir, logger): + rgit = GitInterface(root_dir, logger) + _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + return superroot + +async def submodules_update(gitmodules, root_dir, requiredlist, force): + async def update_submodule(name, requiredlist, force): + submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) + + _, needsupdate, localmods, testfails = submod.status() + if not submod.fxrequired: + submod.fxrequired = "AlwaysRequired" + fxrequired = submod.fxrequired + allowedvalues = fxrequired_allowed_values() + assert fxrequired in allowedvalues + + superroot = git_toplevelroot(root_dir, logger) + + if ( + fxrequired + and ((superroot and "Toplevel" in fxrequired) + or fxrequired not in requiredlist) + ): + if "Optional" in fxrequired and "Optional" not in requiredlist: + if fxrequired.startswith("Always"): + print(f"Skipping optional component {name:>20}") + return # continue to next submodule + optional = "AlwaysOptional" in requiredlist + + if fxrequired in requiredlist: + await submod.update() + repodir = os.path.join(root_dir, submod.path) + if os.path.exists(os.path.join(repodir, ".gitmodules")): + # recursively handle this checkout + print(f"Recursively checking out submodules of {name}") + gitsubmodules = GitModules(submod.logger, confpath=repodir) + newrequiredlist = ["AlwaysRequired"] + if optional: + newrequiredlist.append("AlwaysOptional") + await submodules_update(gitsubmodules, repodir, newrequiredlist, force=force) + + tasks = [update_submodule(name, requiredlist, force) for name in gitmodules.sections()] + await asyncio.gather(*tasks) + +def local_mods_output(): + text = """\ + The submodules labeled with 'M' above are not in a clean state. + The following are options for how to proceed: + (1) Go into each submodule which is not in a clean state and issue a 'git status' + Either revert or commit your changes so that the submodule is in a clean state. + (2) use the --force option to git-fleximod + (3) you can name the particular submodules to update using the git-fleximod command line + (4) As a last resort you can remove the submodule (via 'rm -fr [directory]') + then rerun git-fleximod update. +""" + print(text) + +def submodules_test(gitmodules, root_dir): + """ + This function tests the git submodules based on the provided parameters. + + It first checks that fxtags are present and in sync with submodule hashes. + Then it ensures that urls are consistent with fxurls (not forks and not ssh) + and that sparse checkout files exist. + + Parameters: + gitmodules (ConfigParser): The gitmodules configuration. + root_dir (str): The root directory for the git operation. + + Returns: + int: The number of test failures. + """ + # First check that fxtags are present and in sync with submodule hashes + testfails, localmods, needsupdate = submodules_status(gitmodules, root_dir) + print("") + # Then make sure that urls are consistant with fxurls (not forks and not ssh) + # and that sparse checkout files exist + for name in gitmodules.sections(): + url = gitmodules.get(name, "url") + fxurl = gitmodules.get(name, "fxDONOTUSEurl") + fxsparse = gitmodules.get(name, "fxsparse") + path = gitmodules.get(name, "path") + fxurl = fxurl[:-4] if fxurl.endswith(".git") else fxurl + url = url[:-4] if url.endswith(".git") else url + if not fxurl or url.lower() != fxurl.lower(): + print(f"{name:>20} url {url} not in sync with required {fxurl}") + testfails += 1 + if fxsparse and not os.path.isfile(os.path.join(root_dir, path, fxsparse)): + print(f"{name:>20} sparse checkout file {fxsparse} not found") + testfails += 1 + return testfails + localmods + needsupdate + + +def main(): + ( + root_dir, + file_name, + fxrequired, + includelist, + excludelist, + force, + action, + ) = commandline_arguments() + # Get a logger for the package + global logger + logger = logging.getLogger(__name__) + + logger.info("action is {} root_dir={} file_name={}".format(action, root_dir, file_name)) + + if not root_dir or not os.path.isfile(os.path.join(root_dir, file_name)): + if root_dir: + file_path = utils.find_upwards(root_dir, file_name) + + if root_dir is None or file_path is None: + root_dir = "." + utils.fatal_error( + "No {} found in {} or any of it's parents".format(file_name, root_dir) + ) + + root_dir = os.path.dirname(file_path) + logger.info( + "root_dir is {} includelist={} excludelist={}".format( + root_dir, includelist, excludelist + ) + ) + gitmodules = GitModules( + logger, + confpath=root_dir, + conffile=file_name, + includelist=includelist, + excludelist=excludelist, + ) + if not gitmodules.sections(): + sys.exit(f"No submodule components found, root_dir={root_dir}") + retval = 0 + if action == "update": + asyncio.run(submodules_update(gitmodules, root_dir, fxrequired, force)) + elif action == "status": + tfails, lmods, updates = submodules_status(gitmodules, root_dir, toplevel=True) + if tfails + lmods + updates > 0: + print( + f" testfails = {tfails}, local mods = {lmods}, needs updates {updates}\n" + ) + if lmods > 0: + local_mods_output() + elif action == "test": + retval = submodules_test(gitmodules, root_dir) + else: + utils.fatal_error(f"unrecognized action request {action}") + return retval + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.lib/git_fleximod/gitinterface.py b/.lib/git_fleximod/gitinterface.py new file mode 100644 index 00000000..1a736e4e --- /dev/null +++ b/.lib/git_fleximod/gitinterface.py @@ -0,0 +1,115 @@ +import os +import sys +from . import utils +from pathlib import Path +import asyncio + +class GitInterface: + def __init__(self, repo_path, logger): + logger.debug("Initialize GitInterface for {}".format(repo_path)) + if isinstance(repo_path, str): + self.repo_path = Path(repo_path).resolve() + elif isinstance(repo_path, Path): + self.repo_path = repo_path.resolve() + else: + raise TypeError("repo_path must be a str or Path object") + self.logger = logger + try: + import git + + self._use_module = True + try: + self.repo = git.Repo(str(self.repo_path)) # Initialize GitPython repo + except git.exc.InvalidGitRepositoryError: + self.git = git + self._init_git_repo() + msg = "Using GitPython interface to git" + except ImportError: + self._use_module = False + if not (self.repo_path / ".git").exists(): + self._init_git_repo() + msg = "Using shell interface to git" + self.logger.info(msg) + + def _git_command(self, operation, *args): + self.logger.info(operation) + if self._use_module and operation != "submodule": + try: + return getattr(self.repo.git, operation)(*args) + except Exception as e: + sys.exit(e) + else: + return ["git", "-C", str(self.repo_path), operation] + list(args) + + def _init_git_repo(self): + if self._use_module: + self.repo = self.git.Repo.init(str(self.repo_path)) + else: + command = ("git", "-C", str(self.repo_path), "init") + utils.execute_subprocess(command) + + def _git_operation_command(self, operation, args): + newargs = [] + for a in args: + # Do not use ssh interface + if isinstance(a, str): + a = a.replace("git@github.com:", "https://github.com/") + newargs.append(a) + + return self._git_command(operation, *newargs) + + # pylint: disable=unused-argument + def git_operation(self, operation, *args, **kwargs): + command = self._git_operation_command(operation, args) + if isinstance(command, list): + try: + status, output = utils.execute_subprocess(command, status_to_caller=True, output_to_caller=True) + return status, output.rstrip() + except Exception as e: + sys.exit(e) + else: + return 0, command + + # pylint: disable=unused-argument + async def git_operation_async(self, operation, *args, **kwargs): + command = self._git_operation_command(operation, args) + if isinstance(command, list): + try: + process = await asyncio.create_subprocess_exec( + *command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await process.communicate() + status = process.returncode + output = stdout.decode().strip() if stdout else stderr.decode().strip() + return status, output + except Exception as e: + sys.exit(e) + else: + return 0, command + + def config_get_value(self, section, name): + if self._use_module: + config = self.repo.config_reader() + try: + val = config.get_value(section, name) + except: + val = None + return val + else: + cmd = ("git", "-C", str(self.repo_path), "config", "--get", f"{section}.{name}") + output = utils.execute_subprocess(cmd, output_to_caller=True) + return output.strip() + + def config_set_value(self, section, name, value): + if self._use_module: + with self.repo.config_writer() as writer: + if "." in section: + section = section.replace("."," \"")+'"' + writer.set_value(section, name, value) + writer.release() # Ensure changes are saved + else: + cmd = ("git", "-C", str(self.repo_path), "config", f"{section}.{name}", value) + self.logger.info(cmd) + utils.execute_subprocess(cmd, output_to_caller=True) diff --git a/.lib/git_fleximod/gitmodules.py b/.lib/git_fleximod/gitmodules.py new file mode 100644 index 00000000..cf8b350d --- /dev/null +++ b/.lib/git_fleximod/gitmodules.py @@ -0,0 +1,97 @@ +import shutil, os +from pathlib import Path +from configparser import RawConfigParser, ConfigParser +from .lstripreader import LstripReader + + +class GitModules(RawConfigParser): + def __init__( + self, + logger, + confpath=Path.cwd(), + conffile=".gitmodules", + includelist=None, + excludelist=None, + ): + """ + confpath: Path to the directory containing the .gitmodules file (defaults to the current working directory). + conffile: Name of the configuration file (defaults to .gitmodules). + includelist: Optional list of submodules to include. + excludelist: Optional list of submodules to exclude. + """ + self.logger = logger + self.logger.debug( + "Creating a GitModules object {} {} {} {}".format( + confpath, conffile, includelist, excludelist + ) + ) + super().__init__() + self.conf_file = (Path(confpath) / Path(conffile)) + if self.conf_file.exists(): + self.read_file(LstripReader(str(self.conf_file)), source=conffile) + self.includelist = includelist + self.excludelist = excludelist + self.isdirty = False + + def reload(self): + self.clear() + if self.conf_file.exists(): + self.read_file(LstripReader(str(self.conf_file)), source=self.conf_file) + + + def set(self, name, option, value): + """ + Sets a configuration value for a specific submodule: + Ensures the appropriate section exists for the submodule. + Calls the parent class's set method to store the value. + """ + self.isdirty = True + self.logger.debug("set called {} {} {}".format(name, option, value)) + section = f'submodule "{name}"' + if not self.has_section(section): + self.add_section(section) + super().set(section, option, str(value)) + + # pylint: disable=redefined-builtin, arguments-differ + def get(self, name, option, raw=False, vars=None, fallback=None): + """ + Retrieves a configuration value for a specific submodule: + Uses the parent class's get method to access the value. + Handles potential errors if the section or option doesn't exist. + """ + self.logger.debug("git get called {} {}".format(name, option)) + section = f'submodule "{name}"' + try: + return ConfigParser.get( + self, section, option, raw=raw, vars=vars, fallback=fallback + ) + except ConfigParser.NoOptionError: + return None + + def save(self): + if self.isdirty: + self.logger.info("Writing {}".format(self.conf_file)) + with open(self.conf_file, "w") as fd: + self.write(fd) + self.isdirty = False + + def __del__(self): + self.save() + + def sections(self): + """Strip the submodule part out of section and just use the name""" + self.logger.debug("calling GitModules sections iterator") + names = [] + for section in ConfigParser.sections(self): + name = section[11:-1] + if self.includelist and name not in self.includelist: + continue + if self.excludelist and name in self.excludelist: + continue + names.append(name) + return names + + def items(self, name, raw=False, vars=None): + self.logger.debug("calling GitModules items for {}".format(name)) + section = f'submodule "{name}"' + return ConfigParser.items(section, raw=raw, vars=vars) diff --git a/.lib/git_fleximod/lstripreader.py b/.lib/git_fleximod/lstripreader.py new file mode 100644 index 00000000..01d5580e --- /dev/null +++ b/.lib/git_fleximod/lstripreader.py @@ -0,0 +1,43 @@ +class LstripReader(object): + "LstripReader formats .gitmodules files to be acceptable for configparser" + + def __init__(self, filename): + with open(filename, "r") as infile: + lines = infile.readlines() + self._lines = list() + self._num_lines = len(lines) + self._index = 0 + for line in lines: + self._lines.append(line.lstrip()) + + def readlines(self): + """Return all the lines from this object's file""" + return self._lines + + def readline(self, size=-1): + """Format and return the next line or raise StopIteration""" + try: + line = self.next() + except StopIteration: + line = "" + + if (size > 0) and (len(line) < size): + return line[0:size] + + return line + + def __iter__(self): + """Begin an iteration""" + self._index = 0 + return self + + def next(self): + """Return the next line or raise StopIteration""" + if self._index >= self._num_lines: + raise StopIteration + + self._index = self._index + 1 + return self._lines[self._index - 1] + + def __next__(self): + return self.next() diff --git a/.lib/git_fleximod/metoflexi.py b/.lib/git_fleximod/metoflexi.py new file mode 100755 index 00000000..cc347db2 --- /dev/null +++ b/.lib/git_fleximod/metoflexi.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python +from configparser import ConfigParser +import sys +import shutil +from pathlib import Path +import argparse +import logging +from git_fleximod.gitinterface import GitInterface +from git_fleximod.gitmodules import GitModules +from git_fleximod import utils + +logger = None + +def find_root_dir(filename=".git"): + d = Path.cwd() + root = Path(d.root) + while d != root: + attempt = d / filename + if attempt.is_dir(): + return d + d = d.parent + return None + + +def get_parser(): + description = """ + %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models + """ + parser = argparse.ArgumentParser( + description=description, formatter_class=argparse.RawDescriptionHelpFormatter + ) + + parser.add_argument('-e', '--externals', nargs='?', + default='Externals.cfg', + help='The externals description filename. ' + 'Default: %(default)s.') + + parser.add_argument( + "-C", + "--path", + default=find_root_dir(), + help="Toplevel repository directory. Defaults to top git directory relative to current.", + ) + + parser.add_argument( + "-g", + "--gitmodules", + nargs="?", + default=".gitmodules", + help="The submodule description filename. " "Default: %(default)s.", + ) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Output additional information to " + "the screen and log file. This flag can be " + "used up to two times, increasing the " + "verbosity level each time.", + ) + parser.add_argument( + "-d", + "--debug", + action="store_true", + default=False, + help="DEVELOPER: output additional debugging " + "information to the screen and log file.", + ) + + return parser + +def commandline_arguments(args=None): + parser = get_parser() + + options = parser.parse_args(args) + handlers = [logging.StreamHandler()] + + if options.debug: + try: + open("fleximod.log", "w") + except PermissionError: + sys.exit("ABORT: Could not write file fleximod.log") + level = logging.DEBUG + handlers.append(logging.FileHandler("fleximod.log")) + elif options.verbose: + level = logging.INFO + else: + level = logging.WARNING + # Configure the root logger + logging.basicConfig( + level=level, format="%(name)s - %(levelname)s - %(message)s", handlers=handlers + ) + + return( + options.path, + options.gitmodules, + options.externals + ) + +class ExternalRepoTranslator: + """ + Translates external repositories configured in an INI-style externals file. + """ + + def __init__(self, rootpath, gitmodules, externals): + self.rootpath = rootpath + if gitmodules: + self.gitmodules = GitModules(logger, confpath=rootpath) + self.externals = (rootpath / Path(externals)).resolve() + print(f"Translating {self.externals}") + self.git = GitInterface(rootpath, logger) + +# def __del__(self): +# if (self.rootpath / "save.gitignore"): + + + def translate_single_repo(self, section, tag, url, path, efile, hash_, sparse, protocol): + """ + Translates a single repository based on configuration details. + + Args: + rootpath (str): Root path of the main repository. + gitmodules (str): Path to the .gitmodules file. + tag (str): The tag to use for the external repository. + url (str): The URL of the external repository. + path (str): The relative path within the main repository for the external repository. + efile (str): The external file or file containing submodules. + hash_ (str): The commit hash to checkout (if applicable). + sparse (str): Boolean indicating whether to use sparse checkout (if applicable). + protocol (str): The protocol to use (e.g., 'git', 'http'). + """ + assert protocol != "svn", "SVN protocol is not currently supported" + print(f"Translating repository {section}") + if efile: + file_path = Path(path) / Path(efile) + newroot = (self.rootpath / file_path).parent.resolve() + if not newroot.exists(): + newroot.mkdir(parents=True) + logger.info("Newroot is {}".format(newroot)) + newt = ExternalRepoTranslator(newroot, ".gitmodules", efile) + newt.translate_repo() + if protocol == "externals_only": + if tag: + self.gitmodules.set(section, "fxtag", tag) + if hash_: + self.gitmodules.set(section, "fxtag", hash_) + + self.gitmodules.set(section, "fxDONOTUSEurl", url) + if sparse: + self.gitmodules.set(section, "fxsparse", sparse) + self.gitmodules.set(section, "fxrequired", "ToplevelRequired") + else: + newpath = (self.rootpath / Path(path)) + if newpath.exists(): + shutil.rmtree(newpath) + logger.info("Creating directory {}".format(newpath)) + newpath.mkdir(parents=True) + if tag: + logger.info("cloning {}".format(section)) + try: + self.git.git_operation("clone", "-b", tag, "--depth", "1", url, path) + except: + self.git.git_operation("clone", url, path) + with utils.pushd(newpath): + ngit = GitInterface(newpath, logger) + ngit.git_operation("checkout", tag) + if hash_: + self.git.git_operation("clone", url, path) + git = GitInterface(newpath, logger) + git.git_operation("fetch", "origin") + git.git_operation("checkout", hash_) + if sparse: + print("setting as sparse submodule {}".format(section)) + sparsefile = (newpath / Path(sparse)) + newfile = (newpath / ".git" / "info" / "sparse-checkout") + print(f"sparsefile {sparsefile} newfile {newfile}") + shutil.copy(sparsefile, newfile) + + logger.info("adding submodule {}".format(section)) + self.gitmodules.save() + self.git.git_operation("submodule", "add", "-f", "--name", section, url, path) + self.git.git_operation("submodule","absorbgitdirs") + self.gitmodules.reload() + if tag: + self.gitmodules.set(section, "fxtag", tag) + if hash_: + self.gitmodules.set(section, "fxtag", hash_) + + self.gitmodules.set(section, "fxDONOTUSEurl", url) + if sparse: + self.gitmodules.set(section, "fxsparse", sparse) + self.gitmodules.set(section, "fxrequired", "ToplevelRequired") + + + def translate_repo(self): + """ + Translates external repositories defined within an external file. + + Args: + rootpath (str): Root path of the main repository. + gitmodules (str): Path to the .gitmodules file. + external_file (str): The path to the external file containing repository definitions. + """ + econfig = ConfigParser() + econfig.read((self.rootpath / Path(self.externals))) + + for section in econfig.sections(): + if section == "externals_description": + logger.info("skipping section {}".format(section)) + return + logger.info("Translating section {}".format(section)) + tag = econfig.get(section, "tag", raw=False, fallback=None) + url = econfig.get(section, "repo_url", raw=False, fallback=None) + path = econfig.get(section, "local_path", raw=False, fallback=None) + efile = econfig.get(section, "externals", raw=False, fallback=None) + hash_ = econfig.get(section, "hash", raw=False, fallback=None) + sparse = econfig.get(section, "sparse", raw=False, fallback=None) + protocol = econfig.get(section, "protocol", raw=False, fallback=None) + + self.translate_single_repo(section, tag, url, path, efile, hash_, sparse, protocol) + + + +def _main(): + rootpath, gitmodules, externals = commandline_arguments() + global logger + logger = logging.getLogger(__name__) + with utils.pushd(rootpath): + t = ExternalRepoTranslator(Path(rootpath), gitmodules, externals) + logger.info("Translating {}".format(rootpath)) + t.translate_repo() + + +if __name__ == "__main__": + sys.exit(_main()) diff --git a/.lib/git_fleximod/submodule.py b/.lib/git_fleximod/submodule.py new file mode 100644 index 00000000..572237d0 --- /dev/null +++ b/.lib/git_fleximod/submodule.py @@ -0,0 +1,428 @@ +import os +import textwrap +import shutil +import string +from configparser import NoOptionError +from git_fleximod import utils +from git_fleximod.gitinterface import GitInterface + +class Submodule(): + """ + Represents a Git submodule with enhanced features for flexible management. + + Attributes: + name (str): The name of the submodule. + root_dir (str): The root directory of the main project. + path (str): The relative path from the root directory to the submodule. + url (str): The URL of the submodule repository. + fxurl (str): The URL for flexible submodule management (optional). + fxtag (str): The tag for flexible submodule management (optional). + fxsparse (str): Path to the sparse checkout file relative to the submodule path, see git-sparse-checkout for details (optional). + fxrequired (str): Indicates if the submodule is optional or required (optional). + logger (logging.Logger): Logger instance for logging (optional). + """ + def __init__(self, root_dir, name, path, url, fxtag=None, fxurl=None, fxsparse=None, fxrequired=None, logger=None): + """ + Initializes a new Submodule instance with the provided attributes. + """ + self.name = name + self.root_dir = root_dir + self.path = path + self.url = url + self.fxurl = fxurl + self.fxtag = fxtag + self.fxsparse = fxsparse + if fxrequired: + self.fxrequired = fxrequired + else: + self.fxrequired = "AlwaysRequired" + self.logger = logger + + def status(self): + """ + Checks the status of the submodule and returns 4 parameters: + - result (str): The status of the submodule. + - needsupdate (bool): An indicator if the submodule needs to be updated. + - localmods (bool): An indicator if the submodule has local modifications. + - testfails (bool): An indicator if the submodule has failed a test, this is used for testing purposes. + """ + + smpath = os.path.join(self.root_dir, self.path) + testfails = False + localmods = False + needsupdate = False + ahash = None + optional = "" + if "Optional" in self.fxrequired: + optional = " (optional)" + required = None + level = None + if not os.path.exists(os.path.join(smpath, ".git")): + rootgit = GitInterface(self.root_dir, self.logger) + # submodule commands use path, not name + status, tags = rootgit.git_operation("ls-remote", "--tags", self.url) + status, result = rootgit.git_operation("submodule","status",smpath) + result = result.split() + + if result: + ahash = result[0][1:] + hhash = None + atag = None + for htag in tags.split("\n"): + if htag.endswith('^{}'): + htag = htag[:-3] + if ahash and not atag and ahash in htag: + atag = (htag.split()[1])[10:] + if self.fxtag and not hhash and htag.endswith(self.fxtag): + hhash = htag.split()[0] + if hhash and atag: + break + if self.fxtag and (ahash == hhash or atag == self.fxtag): + result = f"e {self.name:>20} not checked out, aligned at tag {self.fxtag}{optional}" + needsupdate = True + elif self.fxtag: + status, ahash = rootgit.git_operation( + "submodule", "status", "{}".format(self.path) + ) + ahash = ahash[1 : len(self.fxtag) + 1] + if self.fxtag == ahash: + result = f"e {self.name:>20} not checked out, aligned at hash {ahash}{optional}" + else: + result = f"e {self.name:>20} not checked out, out of sync at tag {atag}, expected tag is {self.fxtag}{optional}" + testfails = True + needsupdate = True + else: + result = f"e {self.name:>20} has no fxtag defined in .gitmodules{optional}" + testfails = False + else: + with utils.pushd(smpath): + git = GitInterface(smpath, self.logger) + status, remote = git.git_operation("remote") + if remote == '': + result = f"e {self.name:>20} has no associated remote" + testfails = True + needsupdate = True + return result, needsupdate, localmods, testfails + status, rurl = git.git_operation("ls-remote","--get-url") + status, lines = git.git_operation("log", "--pretty=format:\"%h %d\"") + line = lines.partition('\n')[0] + parts = line.split() + ahash = parts[0][1:] + atag = None + if len(parts) > 3: + idx = 0 + while idx < len(parts)-1: + idx = idx+1 + if parts[idx] == 'tag:': + atag = parts[idx+1] + while atag.endswith(')') or atag.endswith(',') or atag.endswith("\""): + atag = atag[:-1] + if atag == self.fxtag: + break + + + #print(f"line is {line} ahash is {ahash} atag is {atag} {parts}") + # atag = git.git_operation("describe", "--tags", "--always") + # ahash = git.git_operation("rev-list", "HEAD").partition("\n")[0] + + recurse = False + if rurl != self.url: + remote = self._add_remote(git) + git.git_operation("fetch", remote) + if self.fxtag and atag == self.fxtag: + result = f" {self.name:>20} at tag {self.fxtag}" + recurse = True + testfails = False + elif self.fxtag and (ahash[: len(self.fxtag)] == self.fxtag or (self.fxtag.find(ahash)==0)): + result = f" {self.name:>20} at hash {ahash}" + recurse = True + testfails = False + elif atag == ahash: + result = f" {self.name:>20} at hash {ahash}" + recurse = True + elif self.fxtag: + result = f"s {self.name:>20} {atag} {ahash} is out of sync with .gitmodules {self.fxtag}" + testfails = True + needsupdate = True + else: + if atag: + result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {atag}" + else: + result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {ahash}" + testfails = False + + status, output = git.git_operation("status", "--ignore-submodules", "-uno") + if "nothing to commit" not in output: + localmods = True + result = "M" + textwrap.indent(output, " ") +# print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}") + return result, needsupdate, localmods, testfails + + + def _add_remote(self, git): + """ + Adds a new remote to the submodule if it does not already exist. + + This method checks the existing remotes of the submodule. If the submodule's URL is not already listed as a remote, + it attempts to add a new remote. The name for the new remote is generated dynamically to avoid conflicts. If no + remotes exist, it defaults to naming the new remote 'origin'. + + Args: + git (GitInterface): An instance of GitInterface to perform git operations. + + Returns: + str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL. + """ + status, remotes = git.git_operation("remote", "-v") + remotes = remotes.splitlines() + upstream = None + if remotes: + status, upstream = git.git_operation("ls-remote", "--get-url") + newremote = "newremote.00" + tmpurl = self.url.replace("git@github.com:", "https://github.com/") + line = next((s for s in remotes if self.url in s or tmpurl in s), None) + if line: + newremote = line.split()[0] + return newremote + else: + i = 0 + while newremote in remotes: + i = i + 1 + newremote = f"newremote.{i:02d}" + else: + newremote = "origin" + git.git_operation("remote", "add", newremote, self.url) + return newremote + + def toplevel(self): + """ + Returns True if the submodule is Toplevel (either Required or Optional) + """ + return True if "Top" in self.fxrequired else False + + def sparse_checkout(self): + """ + Performs a sparse checkout of the submodule. + + This method optimizes the checkout process by only checking out files specified in the submodule's sparse-checkout configuration, + rather than the entire submodule content. It achieves this by first ensuring the `.git/info/sparse-checkout` file is created and + configured in the submodule's directory. Then, it proceeds to checkout the desired tag. If the submodule has already been checked out, + this method will not perform the checkout again. + + This approach is particularly beneficial for submodules with a large number of files, as it significantly reduces the time and disk space + required for the checkout process by avoiding the unnecessary checkout and subsequent removal of unneeded files. + + Returns: + None + """ + self.logger.info("Called sparse_checkout for {}".format(self.name)) + rgit = GitInterface(self.root_dir, self.logger) + status, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + if superroot: + gitroot = superroot.strip() + else: + gitroot = self.root_dir + # Now need to move the .git dir to the submodule location + rootdotgit = os.path.join(self.root_dir, ".git") + while os.path.isfile(rootdotgit): + with open(rootdotgit) as f: + line = f.readline().rstrip() + if line.startswith("gitdir: "): + rootdotgit = os.path.abspath(os.path.join(self.root_dir,line[8:])) + assert os.path.isdir(rootdotgit) + # first create the module directory + if not os.path.isdir(os.path.join(self.root_dir, self.path)): + os.makedirs(os.path.join(self.root_dir, self.path)) + + # initialize a new git repo and set the sparse checkout flag + sprep_repo = os.path.join(self.root_dir, self.path) + sprepo_git = GitInterface(sprep_repo, self.logger) + if os.path.exists(os.path.join(sprep_repo, ".git")): + try: + self.logger.info("Submodule {} found".format(self.name)) + chk = sprepo_git.config_get_value("core", "sparseCheckout") + if chk == "true": + self.logger.info("Sparse submodule {} already checked out".format(self.name)) + return + except (NoOptionError): + self.logger.debug("Sparse submodule {} not present".format(self.name)) + except Exception as e: + utils.fatal_error("Unexpected error {} occured.".format(e)) + + sprepo_git.config_set_value("core", "sparseCheckout", "true") + + # set the repository remote + + self.logger.info("Setting remote origin in {}/{}".format(self.root_dir, self.path)) + status, remotes = sprepo_git.git_operation("remote", "-v") + if self.url not in remotes: + sprepo_git.git_operation("remote", "add", "origin", self.url) + + topgit = os.path.join(gitroot, ".git") + + if gitroot != self.root_dir and os.path.isfile(os.path.join(self.root_dir, ".git")): + with open(os.path.join(self.root_dir, ".git")) as f: + gitpath = os.path.relpath( + os.path.join(self.root_dir, f.read().split()[1]), + start=os.path.join(self.root_dir, self.path), + ) + rootdotgit = os.path.join(gitpath, "modules", self.name) + else: + rootdotgit = os.path.relpath( + os.path.join(self.root_dir, ".git", "modules", self.name), + start=os.path.join(self.root_dir, self.path), + ) + + if os.path.isdir(os.path.join(self.root_dir, self.path, ".git")): + with utils.pushd(sprep_repo): + if os.path.isdir(os.path.join(rootdotgit,".git")): + shutil.rmtree(os.path.join(rootdotgit,".git")) + shutil.move(".git", rootdotgit) + with open(".git", "w") as f: + f.write("gitdir: " + os.path.relpath(rootdotgit)) + infodir = os.path.join(rootdotgit, "info") + if not os.path.isdir(infodir): + os.makedirs(infodir) + gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout")) + if os.path.isfile(gitsparse): + self.logger.warning( + "submodule {} is already initialized {}".format(self.name, rootdotgit) + ) + return + + with utils.pushd(sprep_repo): + if os.path.isfile(self.fxsparse): + + shutil.copy(self.fxsparse, gitsparse) + + + # Finally checkout the repo + sprepo_git.git_operation("fetch", "origin", "--tags") + status,_ = sprepo_git.git_operation("checkout", self.fxtag) + if status: + print(f"Error checking out {self.name:>20} at {self.fxtag}") + else: + print(f"Successfully checked out {self.name:>20} at {self.fxtag}") + rgit.config_set_value('submodule.' + self.name, "active", "true") + rgit.config_set_value('submodule.' + self.name, "url", self.url) + rgit.config_set_value('submodule.' + self.name, "path", self.path) + + async def update(self): + """ + Updates the submodule to the latest or specified version. + + This method handles the update process of the submodule, including checking out the submodule into the specified path, + handling sparse checkouts if configured, and updating the submodule's URL if necessary. It supports both SSH and HTTPS URLs, + automatically converting SSH URLs to HTTPS to avoid issues for users without SSH keys. + + The update process involves the following steps: + 1. If the submodule is configured for sparse checkout, it performs a sparse checkout. + 2. If the submodule is not already checked out, it clones the submodule using the provided URL. + 3. If a specific tag or hash is provided, it checks out that tag; otherwise, it checks out the latest version. + 4. If the root `.git` is a file (indicating a submodule or a worktree), additional steps are taken to integrate the submodule properly. + + Args: + None + Note: + - SSH URLs are automatically converted to HTTPS to accommodate users without SSH keys. + + Returns: + None + """ + git = GitInterface(self.root_dir, self.logger) + repodir = os.path.join(self.root_dir, self.path) + self.logger.info("Checkout {} into {}/{}".format(self.name, self.root_dir, self.path)) + # if url is provided update to the new url + tag = None + repo_exists = False + if os.path.exists(os.path.join(repodir, ".git")): + self.logger.info("Submodule {} already checked out".format(self.name)) + repo_exists = True + # Look for a .gitmodules file in the newly checkedout repo + if self.fxsparse: + print(f"Sparse checkout {self.name} fxsparse {self.fxsparse}") + self.sparse_checkout() + else: + if not repo_exists and self.url: + # ssh urls cause problems for those who dont have git accounts with ssh keys defined + # but cime has one since e3sm prefers ssh to https, because the .gitmodules file was + # opened with a GitModules object we don't need to worry about restoring the file here + # it will be done by the GitModules class + if self.url.startswith("git@"): + git.git_operation("clone", self.url, self.path) + smgit = GitInterface(repodir, self.logger) + if not tag: + status, tag = smgit.git_operation("describe", "--tags", "--always") + smgit.git_operation("checkout", tag) + # Now need to move the .git dir to the submodule location + rootdotgit = os.path.join(self.root_dir, ".git") + if os.path.isfile(rootdotgit): + with open(rootdotgit) as f: + line = f.readline() + if line.startswith("gitdir: "): + rootdotgit = line[8:] + + newpath = os.path.abspath(os.path.join(self.root_dir, rootdotgit, "modules", self.name)) + if os.path.exists(newpath): + shutil.rmtree(os.path.join(repodir, ".git")) + else: + shutil.move(os.path.join(repodir, ".git"), newpath) + + with open(os.path.join(repodir, ".git"), "w") as f: + f.write("gitdir: " + os.path.relpath(newpath, start=repodir)) + + if not os.path.exists(repodir): + parent = os.path.dirname(repodir) + if not os.path.isdir(parent): + os.makedirs(parent) + git.git_operation("submodule", "add", "--name", self.name, "--", self.url, self.path) + + if not repo_exists: + git.git_operation("submodule", "init", "--", self.path) + await git.git_operation_async("submodule", "update", "--", self.path) + + if self.fxtag: + smgit = GitInterface(repodir, self.logger) + newremote = self._add_remote(smgit) + # Trying to distingush a tag from a hash + allowed = set(string.digits + 'abcdef') + if not set(self.fxtag) <= allowed: + # This is a tag + tag = f"refs/tags/{self.fxtag}:refs/tags/{self.fxtag}" + smgit.git_operation("fetch", newremote, tag) + smgit.git_operation("checkout", self.fxtag) + + if not os.path.exists(os.path.join(repodir, ".git")): + utils.fatal_error( + f"Failed to checkout {self.name} {repo_exists} {repodir} {self.path}" + ) + + + if os.path.exists(os.path.join(self.path, ".git")): + submoddir = os.path.join(self.root_dir, self.path) + with utils.pushd(submoddir): + git = GitInterface(submoddir, self.logger) + # first make sure the url is correct + newremote = self._add_remote(git) + status, tags = git.git_operation("tag", "-l") + fxtag = self.fxtag + if fxtag and fxtag not in tags: + git.git_operation("fetch", newremote, "--tags") + status, atag = git.git_operation("describe", "--tags", "--always") + if fxtag and fxtag != atag: + try: + status, _ = git.git_operation("checkout", fxtag) + if not status: + print(f"{self.name:>20} updated to {fxtag}") + except Exception as error: + print(error) + + + elif not fxtag: + print(f"No fxtag found for submodule {self.name:>20}") + else: + print(f"{self.name:>20} up to date.") + + + + return diff --git a/.lib/git_fleximod/utils.py b/.lib/git_fleximod/utils.py new file mode 100644 index 00000000..c4f43d52 --- /dev/null +++ b/.lib/git_fleximod/utils.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +""" +Common public utilities for manic package + +""" + +import logging +import os +import subprocess +import sys +from threading import Timer +from pathlib import Path + +LOCAL_PATH_INDICATOR = "." +# --------------------------------------------------------------------- +# +# functions to massage text for output and other useful utilities +# +# --------------------------------------------------------------------- +from contextlib import contextmanager + + +@contextmanager +def pushd(new_dir): + """context for chdir. usage: with pushd(new_dir)""" + previous_dir = os.getcwd() + os.chdir(new_dir) + try: + yield + finally: + os.chdir(previous_dir) + + +def log_process_output(output): + """Log each line of process output at debug level so it can be + filtered if necessary. By default, output is a single string, and + logging.debug(output) will only put log info heading on the first + line. This makes it hard to filter with grep. + + """ + output = output.split("\n") + for line in output: + logging.debug(line) + + +def printlog(msg, **kwargs): + """Wrapper script around print to ensure that everything printed to + the screen also gets logged. + + """ + logging.info(msg) + if kwargs: + print(msg, **kwargs) + else: + print(msg) + sys.stdout.flush() + + +def find_upwards(root_dir, filename): + """Find a file in root dir or any of it's parents""" + d = Path(root_dir) + root = Path(d.root) + while d != root: + attempt = d / filename + if attempt.exists(): + return attempt + d = d.parent + return None + + +def last_n_lines(the_string, n_lines, truncation_message=None): + """Returns the last n lines of the given string + + Args: + the_string: str + n_lines: int + truncation_message: str, optional + + Returns a string containing the last n lines of the_string + + If truncation_message is provided, the returned string begins with + the given message if and only if the string is greater than n lines + to begin with. + """ + + lines = the_string.splitlines(True) + if len(lines) <= n_lines: + return_val = the_string + else: + lines_subset = lines[-n_lines:] + str_truncated = "".join(lines_subset) + if truncation_message: + str_truncated = truncation_message + "\n" + str_truncated + return_val = str_truncated + + return return_val + + +def indent_string(the_string, indent_level): + """Indents the given string by a given number of spaces + + Args: + the_string: str + indent_level: int + + Returns a new string that is the same as the_string, except that + each line is indented by 'indent_level' spaces. + + In python3, this can be done with textwrap.indent. + """ + + lines = the_string.splitlines(True) + padding = " " * indent_level + lines_indented = [padding + line for line in lines] + return "".join(lines_indented) + + +# --------------------------------------------------------------------- +# +# error handling +# +# --------------------------------------------------------------------- + + +def fatal_error(message): + """ + Error output function + """ + logging.error(message) + raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message)) + + +# --------------------------------------------------------------------- +# +# Data conversion / manipulation +# +# --------------------------------------------------------------------- +def str_to_bool(bool_str): + """Convert a sting representation of as boolean into a true boolean. + + Conversion should be case insensitive. + """ + value = None + str_lower = bool_str.lower() + if str_lower in ("true", "t"): + value = True + elif str_lower in ("false", "f"): + value = False + if value is None: + msg = ( + 'ERROR: invalid boolean string value "{0}". ' + 'Must be "true" or "false"'.format(bool_str) + ) + fatal_error(msg) + return value + + +REMOTE_PREFIXES = ["http://", "https://", "ssh://", "git@"] + + +def is_remote_url(url): + """check if the user provided a local file path instead of a + remote. If so, it must be expanded to an absolute + path. + + """ + remote_url = False + for prefix in REMOTE_PREFIXES: + if url.startswith(prefix): + remote_url = True + return remote_url + + +def split_remote_url(url): + """check if the user provided a local file path or a + remote. If remote, try to strip off protocol info. + + """ + remote_url = is_remote_url(url) + if not remote_url: + return url + + for prefix in REMOTE_PREFIXES: + url = url.replace(prefix, "") + + if "@" in url: + url = url.split("@")[1] + + if ":" in url: + url = url.split(":")[1] + + return url + + +def expand_local_url(url, field): + """check if the user provided a local file path instead of a + remote. If so, it must be expanded to an absolute + path. + + Note: local paths of LOCAL_PATH_INDICATOR have special meaning and + represent local copy only, don't work with the remotes. + + """ + remote_url = is_remote_url(url) + if not remote_url: + if url.strip() == LOCAL_PATH_INDICATOR: + pass + else: + url = os.path.expandvars(url) + url = os.path.expanduser(url) + if not os.path.isabs(url): + msg = ( + 'WARNING: Externals description for "{0}" contains a ' + "url that is not remote and does not expand to an " + "absolute path. Version control operations may " + "fail.\n\nurl={1}".format(field, url) + ) + printlog(msg) + else: + url = os.path.normpath(url) + return url + + +# --------------------------------------------------------------------- +# +# subprocess +# +# --------------------------------------------------------------------- + +# Give the user a helpful message if we detect that a command seems to +# be hanging. +_HANGING_SEC = 300 + + +def _hanging_msg(working_directory, command): + print( + """ + +Command '{command}' +from directory {working_directory} +has taken {hanging_sec} seconds. It may be hanging. + +The command will continue to run, but you may want to abort +git-fleximod with ^C and investigate. A possible cause of hangs is git +requires authentication to access a private repository. On some +systems, git requests for authentication information will not +be displayed to the user. In this case, the program will appear to +hang. Ensure you can run git manually and access all +repositories without entering your authentication information. + +""".format( + command=command, + working_directory=working_directory, + hanging_sec=_HANGING_SEC, + ) + ) + + +def execute_subprocess(commands, status_to_caller=False, output_to_caller=False): + """Wrapper around subprocess.check_output to handle common + exceptions. + + check_output runs a command with arguments and waits + for it to complete. + + check_output raises an exception on a nonzero return code. if + status_to_caller is true, execute_subprocess returns the subprocess + return code, otherwise execute_subprocess treats non-zero return + status as an error and raises an exception. + + """ + cwd = os.getcwd() + msg = "In directory: {0}\nexecute_subprocess running command:".format(cwd) + logging.info(msg) + commands_str = " ".join(str(element) for element in commands) + logging.info(commands_str) + return_to_caller = status_to_caller or output_to_caller + status = -1 + output = "" + hanging_timer = Timer( + _HANGING_SEC, + _hanging_msg, + kwargs={"working_directory": cwd, "command": commands_str}, + ) + hanging_timer.start() + try: + output = subprocess.check_output( + commands, stderr=subprocess.STDOUT, universal_newlines=True + ) + log_process_output(output) + status = 0 + except OSError as error: + msg = failed_command_msg( + "Command execution failed. Does the executable exist?", commands + ) + logging.error(error) + fatal_error(msg) + except ValueError as error: + msg = failed_command_msg( + "DEV_ERROR: Invalid arguments trying to run subprocess", commands + ) + logging.error(error) + fatal_error(msg) + except subprocess.CalledProcessError as error: + # Only report the error if we are NOT returning to the + # caller. If we are returning to the caller, then it may be a + # simple status check. If returning, it is the callers + # responsibility determine if an error occurred and handle it + # appropriately. + msg_context = ( + "Process did not run successfully; " + "returned status {0}".format(error.returncode) + ) + msg = failed_command_msg(msg_context, commands, output=error.output) + if not return_to_caller: + logging.error(error) + logging.error(msg) + log_process_output(error.output) + fatal_error(msg) + status = error.returncode + finally: + hanging_timer.cancel() + + if status_to_caller and output_to_caller: + ret_value = (status, output) + elif status_to_caller: + ret_value = status + elif output_to_caller: + ret_value = output + else: + ret_value = None + + return ret_value + + +def failed_command_msg(msg_context, command, output=None): + """Template for consistent error messages from subprocess calls. + + If 'output' is given, it should provide the output from the failed + command + """ + + if output: + output_truncated = last_n_lines( + output, 20, truncation_message="[... Output truncated for brevity ...]" + ) + errmsg = ( + "Failed with output:\n" + indent_string(output_truncated, 4) + "\nERROR: " + ) + else: + errmsg = "" + + command_str = " ".join(command) + errmsg += """In directory + {cwd} +{context}: + {command} +""".format( + cwd=os.getcwd(), context=msg_context, command=command_str + ) + + if output: + errmsg += "See above for output from failed command.\n" + + return errmsg diff --git a/.lib/poetry.lock b/.lib/poetry.lock new file mode 100644 index 00000000..ac82fb0d --- /dev/null +++ b/.lib/poetry.lock @@ -0,0 +1,693 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "alabaster" +version = "0.7.13" +description = "A configurable sidebar-enabled Sphinx theme" +optional = false +python-versions = ">=3.6" +files = [ + {file = "alabaster-0.7.13-py3-none-any.whl", hash = "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3"}, + {file = "alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"}, +] + +[[package]] +name = "babel" +version = "2.15.0" +description = "Internationalization utilities" +optional = false +python-versions = ">=3.8" +files = [ + {file = "Babel-2.15.0-py3-none-any.whl", hash = "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb"}, + {file = "babel-2.15.0.tar.gz", hash = "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"}, +] + +[package.dependencies] +pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} + +[package.extras] +dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] + +[[package]] +name = "certifi" +version = "2024.8.30" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "docutils" +version = "0.19" +description = "Docutils -- Python Documentation Utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, + {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.1" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "fsspec" +version = "2023.12.2" +description = "File-system specification" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fsspec-2023.12.2-py3-none-any.whl", hash = "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"}, + {file = "fsspec-2023.12.2.tar.gz", hash = "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb"}, +] + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +devel = ["pytest", "pytest-cov"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + +[[package]] +name = "gitdb" +version = "4.0.11" +description = "Git Object Database" +optional = false +python-versions = ">=3.7" +files = [ + {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, + {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, +] + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.43" +description = "GitPython is a Python library used to interact with Git repositories" +optional = false +python-versions = ">=3.7" +files = [ + {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, + {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, +] + +[package.dependencies] +gitdb = ">=4.0.1,<5" + +[package.extras] +doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] + +[[package]] +name = "idna" +version = "3.7" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, +] + +[[package]] +name = "imagesize" +version = "1.4.1" +description = "Getting image size from png/jpeg/jpeg2000/gif file" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, + {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, +] + +[[package]] +name = "importlib-metadata" +version = "8.0.0" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_metadata-8.0.0-py3-none-any.whl", hash = "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f"}, + {file = "importlib_metadata-8.0.0.tar.gz", hash = "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812"}, +] + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "jinja2" +version = "3.1.4" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, + {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "24.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pyfakefs" +version = "5.5.0" +description = "pyfakefs implements a fake file system that mocks the Python file system modules." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyfakefs-5.5.0-py3-none-any.whl", hash = "sha256:8dbf203ab7bef1529f11f7d41b9478b898e95bf9f3b71262163aac07a518cd76"}, + {file = "pyfakefs-5.5.0.tar.gz", hash = "sha256:7448aaa07142f892d0a4eb52a5ed3206a9f02c6599e686cd97d624c18979c154"}, +] + +[[package]] +name = "pygments" +version = "2.18.0" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, + {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pytest" +version = "8.2.2" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, + {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2.0" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "smmap" +version = "5.0.1" +description = "A pure Python implementation of a sliding window memory map manager" +optional = false +python-versions = ">=3.7" +files = [ + {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, + {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, +] + +[[package]] +name = "snowballstemmer" +version = "2.2.0" +description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." +optional = false +python-versions = "*" +files = [ + {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, + {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, +] + +[[package]] +name = "sphinx" +version = "5.3.0" +description = "Python documentation generator" +optional = false +python-versions = ">=3.6" +files = [ + {file = "Sphinx-5.3.0.tar.gz", hash = "sha256:51026de0a9ff9fc13c05d74913ad66047e104f56a129ff73e174eb5c3ee794b5"}, + {file = "sphinx-5.3.0-py3-none-any.whl", hash = "sha256:060ca5c9f7ba57a08a1219e547b269fadf125ae25b06b9fa7f66768efb652d6d"}, +] + +[package.dependencies] +alabaster = ">=0.7,<0.8" +babel = ">=2.9" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +docutils = ">=0.14,<0.20" +imagesize = ">=1.3" +importlib-metadata = {version = ">=4.8", markers = "python_version < \"3.10\""} +Jinja2 = ">=3.0" +packaging = ">=21.0" +Pygments = ">=2.12" +requests = ">=2.5.0" +snowballstemmer = ">=2.0" +sphinxcontrib-applehelp = "*" +sphinxcontrib-devhelp = "*" +sphinxcontrib-htmlhelp = ">=2.0.0" +sphinxcontrib-jsmath = "*" +sphinxcontrib-qthelp = "*" +sphinxcontrib-serializinghtml = ">=1.1.5" + +[package.extras] +docs = ["sphinxcontrib-websupport"] +lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-bugbear", "flake8-comprehensions", "flake8-simplify", "isort", "mypy (>=0.981)", "sphinx-lint", "types-requests", "types-typed-ast"] +test = ["cython", "html5lib", "pytest (>=4.6)", "typed_ast"] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "1.0.4" +description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" +optional = false +python-versions = ">=3.8" +files = [ + {file = "sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e"}, + {file = "sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "1.0.2" +description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." +optional = false +python-versions = ">=3.5" +files = [ + {file = "sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"}, + {file = "sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.0.1" +description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff"}, + {file = "sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["html5lib", "pytest"] + +[[package]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +description = "A sphinx extension which renders display math in HTML via JavaScript" +optional = false +python-versions = ">=3.5" +files = [ + {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, + {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, +] + +[package.extras] +test = ["flake8", "mypy", "pytest"] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "1.0.3" +description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." +optional = false +python-versions = ">=3.5" +files = [ + {file = "sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72"}, + {file = "sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "1.1.5" +description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." +optional = false +python-versions = ">=3.5" +files = [ + {file = "sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952"}, + {file = "sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["pytest"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "urllib3" +version = "2.2.2" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wheel" +version = "0.42.0" +description = "A built-package format for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "wheel-0.42.0-py3-none-any.whl", hash = "sha256:177f9c9b0d45c47873b619f5b650346d632cdc35fb5e4d25058e09c9e581433d"}, + {file = "wheel-0.42.0.tar.gz", hash = "sha256:c45be39f7882c9d34243236f2d63cbd58039e360f85d0913425fbd7ceea617a8"}, +] + +[package.extras] +test = ["pytest (>=6.0.0)", "setuptools (>=65)"] + +[[package]] +name = "zipp" +version = "3.19.2" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, + {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, +] + +[package.extras] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8" +content-hash = "25ee2ae1d74abedde3a6637a60d4a3095ea5cf9731960875741bbc2ba84a475d" diff --git a/.lib/pyproject.toml b/.lib/pyproject.toml new file mode 100644 index 00000000..76ef0ef8 --- /dev/null +++ b/.lib/pyproject.toml @@ -0,0 +1,42 @@ +[tool.poetry] +name = "git-fleximod" +version = "1.0.0" +description = "Extended support for git-submodule and git-sparse-checkout" +authors = ["Jim Edwards "] +maintainers = ["Jim Edwards "] +license = "MIT" +readme = "README.md" +homepage = "https://github.com/jedwards4b/git-fleximod" +keywords = ["git", "submodule", "sparse-checkout"] +packages = [ +{ include = "git_fleximod"}, +{ include = "doc"}, +{ include = "README.md"}, +] + +[tool.poetry.scripts] +git-fleximod = "git_fleximod.git_fleximod:main" +me2flexi = "git_fleximod.metoflexi:_main" +fsspec = "fsspec.fuse:main" + +[tool.poetry.dependencies] +python = "^3.8" +GitPython = "^3.1.0" +sphinx = "^5.0.0" +fsspec = "^2023.12.2" +wheel = "^0.42.0" +pytest = "^8.0.0" +pyfakefs = "^5.3.5" + +[tool.poetry.urls] +"Bug Tracker" = "https://github.com/jedwards4b/git-fleximod/issues" + +[tool.pytest.ini_options] +markers = [ + "skip_after_first: only run on first iteration" +] + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + diff --git a/.lib/tbump.toml b/.lib/tbump.toml new file mode 100644 index 00000000..4ac513dd --- /dev/null +++ b/.lib/tbump.toml @@ -0,0 +1,43 @@ +# Uncomment this if your project is hosted on GitHub: +github_url = "https://github.com/jedwards4b/git-fleximod/" + +[version] +current = "1.0.0" + +# Example of a semver regexp. +# Make sure this matches current_version before +# using tbump +regex = ''' + (?P\d+) + \. + (?P\d+) + \. + (?P\d+) + ''' + +[git] +message_template = "Bump to {new_version}" +tag_template = "v{new_version}" + +# For each file to patch, add a [[file]] config +# section containing the path of the file, relative to the +# tbump.toml location. +[[file]] +src = "git_fleximod/cli.py" + +[[file]] +src = "pyproject.toml" + +# You can specify a list of commands to +# run after the files have been patched +# and before the git commit is made + +# [[before_commit]] +# name = "check changelog" +# cmd = "grep -q {new_version} Changelog.rst" + +# Or run some commands after the git tag and the branch +# have been pushed: +# [[after_push]] +# name = "publish" +# cmd = "./publish.sh" diff --git a/.lib/tests/__init__.py b/.lib/tests/__init__.py new file mode 100644 index 00000000..4d4c66c7 --- /dev/null +++ b/.lib/tests/__init__.py @@ -0,0 +1,3 @@ +import sys, os + +sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir, "src")) diff --git a/.lib/tests/conftest.py b/.lib/tests/conftest.py new file mode 100644 index 00000000..44d28e17 --- /dev/null +++ b/.lib/tests/conftest.py @@ -0,0 +1,150 @@ +import pytest +from git_fleximod.gitinterface import GitInterface +import os +import subprocess +import logging +from pathlib import Path + +@pytest.fixture(scope='session') +def logger(): + logging.basicConfig( + level=logging.INFO, format="%(name)s - %(levelname)s - %(message)s", handlers=[logging.StreamHandler()] + ) + logger = logging.getLogger(__name__) + return logger + +all_repos=[ + {"subrepo_path": "modules/test", + "submodule_name": "test_submodule", + "status1" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", + "status2" : "test_submodule at tag MPIserial_2.4.0", + "status3" : "test_submodule at tag MPIserial_2.4.0", + "status4" : "test_submodule at tag MPIserial_2.4.0", + "gitmodules_content" : """ + [submodule "test_submodule"] + path = modules/test + url = https://github.com/ESMCI/mpi-serial.git + fxtag = MPIserial_2.4.0 + fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = ToplevelRequired +"""}, + {"subrepo_path": "modules/test_optional", + "submodule_name": "test_optional", + "status1" : "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", + "status2" : "test_optional at tag MPIserial_2.4.0", + "status3" : "test_optional not checked out, out of sync at tag MPIserial_2.5.4, expected tag is MPIserial_2.4.0 (optional)", + "status4" : "test_optional at tag MPIserial_2.4.0", + "gitmodules_content": """ + [submodule "test_optional"] + path = modules/test_optional + url = https://github.com/ESMCI/mpi-serial.git + fxtag = MPIserial_2.4.0 + fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = ToplevelOptional +"""}, + {"subrepo_path": "modules/test_alwaysoptional", + "submodule_name": "test_alwaysoptional", + "status1" : "test_alwaysoptional MPIserial_2.3.0 is out of sync with .gitmodules e5cf35c", + "status2" : "test_alwaysoptional at hash e5cf35c", + "status3" : "out of sync at tag MPIserial_2.5.4, expected tag is e5cf35c", + "status4" : "test_alwaysoptional at hash e5cf35c", + "gitmodules_content": """ + [submodule "test_alwaysoptional"] + path = modules/test_alwaysoptional + url = https://github.com/ESMCI/mpi-serial.git + fxtag = e5cf35c + fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = AlwaysOptional +"""}, + {"subrepo_path": "modules/test_sparse", + "submodule_name": "test_sparse", + "status1" : "test_sparse at tag MPIserial_2.5.0", + "status2" : "test_sparse at tag MPIserial_2.5.0", + "status3" : "test_sparse at tag MPIserial_2.5.0", + "status4" : "test_sparse at tag MPIserial_2.5.0", + "gitmodules_content": """ + [submodule "test_sparse"] + path = modules/test_sparse + url = https://github.com/ESMCI/mpi-serial.git + fxtag = MPIserial_2.5.0 + fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = AlwaysRequired + fxsparse = ../.sparse_file_list +"""}, +] +@pytest.fixture(params=all_repos) + +def shared_repos(request): + return request.param + +@pytest.fixture +def get_all_repos(): + return all_repos + +def write_sparse_checkout_file(fp): + sparse_content = """m4 +""" + fp.write_text(sparse_content) + +@pytest.fixture +def test_repo(shared_repos, tmp_path, logger): + subrepo_path = shared_repos["subrepo_path"] + submodule_name = shared_repos["submodule_name"] + test_dir = tmp_path / "testrepo" + test_dir.mkdir() + str_path = str(test_dir) + gitp = GitInterface(str_path, logger) + assert test_dir.joinpath(".git").is_dir() + (test_dir / "modules").mkdir() + if "sparse" in submodule_name: + (test_dir / subrepo_path).mkdir() + # Add the sparse checkout file + write_sparse_checkout_file(test_dir / "modules" / ".sparse_file_list") + gitp.git_operation("add","modules/.sparse_file_list") + else: + gitp = GitInterface(str(test_dir), logger) + gitp.git_operation("submodule", "add", "--depth","1","--name", submodule_name, "https://github.com/ESMCI/mpi-serial.git", subrepo_path) + assert test_dir.joinpath(".gitmodules").is_file() + gitp.git_operation("add",subrepo_path) + gitp.git_operation("commit","-a","-m","\"add submod\"") + test_dir2 = tmp_path / "testrepo2" + gitp.git_operation("clone",test_dir,test_dir2) + return test_dir2 + + +@pytest.fixture +def complex_repo(tmp_path, logger): + test_dir = tmp_path / "testcomplex" + test_dir.mkdir() + str_path = str(test_dir) + gitp = GitInterface(str_path, logger) + gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") + gitp.git_operation("fetch", "origin") + gitp.git_operation("checkout", "v0.0.1") + return test_dir + +@pytest.fixture +def complex_update(tmp_path, logger): + test_dir = tmp_path / "testcomplex" + test_dir.mkdir() + str_path = str(test_dir) + gitp = GitInterface(str_path, logger) + gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") + gitp.git_operation("fetch", "origin") + gitp.git_operation("checkout", "v0.0.2") + + return test_dir + +@pytest.fixture +def git_fleximod(): + def _run_fleximod(path, args, input=None): + cmd = ["git", "fleximod"] + args.split() + result = subprocess.run(cmd, cwd=path, input=input, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True) + if result.returncode: + print(result.stdout) + print(result.stderr) + return result + return _run_fleximod + diff --git a/.lib/tests/test_a_import.py b/.lib/tests/test_a_import.py new file mode 100644 index 00000000..d5ca878d --- /dev/null +++ b/.lib/tests/test_a_import.py @@ -0,0 +1,8 @@ +# pylint: disable=unused-import +from git_fleximod import cli +from git_fleximod import utils +from git_fleximod.gitinterface import GitInterface +from git_fleximod.gitmodules import GitModules + +def test_import(): + print("here") diff --git a/.lib/tests/test_b_update.py b/.lib/tests/test_b_update.py new file mode 100644 index 00000000..159f1cfa --- /dev/null +++ b/.lib/tests/test_b_update.py @@ -0,0 +1,26 @@ +import pytest +from pathlib import Path + +def test_basic_checkout(git_fleximod, test_repo, shared_repos): + # Prepare a simple .gitmodules + gm = shared_repos['gitmodules_content'] + file_path = (test_repo / ".gitmodules") + repo_name = shared_repos["submodule_name"] + repo_path = shared_repos["subrepo_path"] + + file_path.write_text(gm) + + # Run the command + result = git_fleximod(test_repo, f"update {repo_name}") + + # Assertions + assert result.returncode == 0 + assert Path(test_repo / repo_path).exists() # Did the submodule directory get created? + if "sparse" in repo_name: + assert Path(test_repo / f"{repo_path}/m4").exists() # Did the submodule sparse directory get created? + assert not Path(test_repo / f"{repo_path}/README").exists() # Did only the submodule sparse directory get created? + + status = git_fleximod(test_repo, f"status {repo_name}") + + assert shared_repos["status2"] in status.stdout + diff --git a/.lib/tests/test_c_required.py b/.lib/tests/test_c_required.py new file mode 100644 index 00000000..89ab8d29 --- /dev/null +++ b/.lib/tests/test_c_required.py @@ -0,0 +1,30 @@ +import pytest +from pathlib import Path + +def test_required(git_fleximod, test_repo, shared_repos): + file_path = (test_repo / ".gitmodules") + gm = shared_repos["gitmodules_content"] + repo_name = shared_repos["submodule_name"] + if file_path.exists(): + with file_path.open("r") as f: + gitmodules_content = f.read() + # add the entry if it does not exist + if repo_name not in gitmodules_content: + file_path.write_text(gitmodules_content+gm) + # or if it is incomplete + elif gm not in gitmodules_content: + file_path.write_text(gm) + else: + file_path.write_text(gm) + result = git_fleximod(test_repo, "update") + assert result.returncode == 0 + status = git_fleximod(test_repo, f"status {repo_name}") + assert shared_repos["status3"] in status.stdout + status = git_fleximod(test_repo, f"update --optional") + assert result.returncode == 0 + status = git_fleximod(test_repo, f"status {repo_name}") + assert shared_repos["status4"] in status.stdout + status = git_fleximod(test_repo, f"update {repo_name}") + assert result.returncode == 0 + status = git_fleximod(test_repo, f"status {repo_name}") + assert shared_repos["status4"] in status.stdout diff --git a/.lib/tests/test_d_complex.py b/.lib/tests/test_d_complex.py new file mode 100644 index 00000000..edde7d81 --- /dev/null +++ b/.lib/tests/test_d_complex.py @@ -0,0 +1,66 @@ +import pytest +from pathlib import Path +from git_fleximod.gitinterface import GitInterface + +def test_complex_checkout(git_fleximod, complex_repo, logger): + status = git_fleximod(complex_repo, "status") + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout) + assert("Complex not checked out, aligned at tag testtag02" in status.stdout) + assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) + + # This should checkout and update test_submodule and complex_sub + result = git_fleximod(complex_repo, "update") + assert result.returncode == 0 + + status = git_fleximod(complex_repo, "status") + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag02" in status.stdout) + + # now check the complex_sub + root = (complex_repo / "modules" / "complex") + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serial" / ".git").exists()) + assert(not (root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + # update a single optional submodule + + result = git_fleximod(complex_repo, "update ToplevelOptional") + assert result.returncode == 0 + + status = git_fleximod(complex_repo, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag02" in status.stdout) + assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) + + # Finally update optional + result = git_fleximod(complex_repo, "update --optional") + assert result.returncode == 0 + + status = git_fleximod(complex_repo, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag02" in status.stdout) + assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout) + + # now check the complex_sub + root = (complex_repo / "modules" / "complex" ) + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + diff --git a/.lib/tests/test_e_complex_update.py b/.lib/tests/test_e_complex_update.py new file mode 100644 index 00000000..0c3ab4c6 --- /dev/null +++ b/.lib/tests/test_e_complex_update.py @@ -0,0 +1,69 @@ +import pytest +from pathlib import Path +from git_fleximod.gitinterface import GitInterface + +def test_complex_update(git_fleximod, complex_update, logger): + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout) + assert("Complex not checked out, out of sync at tag testtag02, expected tag is testtag3" in status.stdout) + assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) + + # This should checkout and update test_submodule and complex_sub + result = git_fleximod(complex_update, "update") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag3" in status.stdout) + + # now check the complex_sub + root = (complex_update / "modules" / "complex") + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serialAR" / ".git").exists()) + assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) + assert(not (root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + # update a single optional submodule + + result = git_fleximod(complex_update, "update ToplevelOptional") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag3" in status.stdout) + assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) + + # Finally update optional + result = git_fleximod(complex_update, "update --optional") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag3" in status.stdout) + assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout) + + # now check the complex_sub + root = (complex_update / "modules" / "complex" ) + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert(not (root / "modules" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serialAR" / ".git").exists()) + assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 1a059472..a22d70ff 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 1a05947243b74b88b5cd669e233071d30ec1024b +Subproject commit a22d70ffa886510ed3e254942cf9e61c1ec91413 From 2a58a1e2a1702d53d38e20af7b85d4a06754490d Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Tue, 15 Jul 2025 10:40:35 -0600 Subject: [PATCH 006/150] latest atmos phys hash --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index fed1b04c..9ba0fc3a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = a22d70ffa886510ed3e254942cf9e61c1ec91413 + fxtag = ac661eb60c3f6b36ce4892b007cd53977d323ee2 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] From 0b817f308f2191d0a8858c5d6a25460738c2b200 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Thu, 7 Aug 2025 13:29:53 -0600 Subject: [PATCH 007/150] lw answers match cam for 2 timesteps using radiation --- .gitmodules | 2 +- cime_config/create_readnl_files.py | 2 +- src/control/cam_comp.F90 | 11 +++- src/control/runtime_opts.F90 | 2 + src/data/cam_var_init_marks.inc | 12 +++- src/data/generate_registry_data.py | 2 +- src/data/registry.xml | 80 ++++++++++++++++++++++- src/data/registry_v1_0.xsd | 2 +- src/physics/ncar_ccpp | 2 +- src/physics/utils/radiation_namelist.F90 | 25 +++---- src/physics/utils/radiation_namelist.meta | 2 +- src/utils/machine.F90 | 12 ++++ 12 files changed, 132 insertions(+), 22 deletions(-) create mode 100644 src/utils/machine.F90 diff --git a/.gitmodules b/.gitmodules index 1ad5575a..feeb8128 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = a3df1acde956130a566460604ae65fa9c2219d9c + fxtag = 67b812d629350f76ecafbc41c7adce04d7b41abb fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/cime_config/create_readnl_files.py b/cime_config/create_readnl_files.py index 1dd5e932..59aa3e96 100644 --- a/cime_config/create_readnl_files.py +++ b/cime_config/create_readnl_files.py @@ -483,7 +483,7 @@ def write_dimension_decls(self, ofile, indent): """ for index, alen in enumerate(self.array_len): aname = self.__array_names[index] - ofile.write(f"integer, parameter :: {aname} = {alen}", indent) + ofile.write(f"integer, public, parameter :: {aname} = {alen}", indent) # end for @property diff --git a/src/control/cam_comp.F90 b/src/control/cam_comp.F90 index afd2adb2..c5ad8fcf 100644 --- a/src/control/cam_comp.F90 +++ b/src/control/cam_comp.F90 @@ -28,7 +28,7 @@ module cam_comp use camsrfexch, only: cam_out_t, cam_in_t use physics_types, only: phys_state, phys_tend use physics_types, only: dtime_phys - use physics_types, only: calday + use physics_types, only: calday, next_calday, radiation_offset, nextsw_cday use physics_types, only: is_first_timestep, nstep use dyn_comp, only: dyn_import_t, dyn_export_t @@ -189,8 +189,12 @@ subroutine cam_init(caseid, ctitle, model_doi_url, & ! Get current fractional calendar day. Needs to be updated at every timestep. calday = get_curr_calday() next_calday = get_curr_calday(offset=int(get_step_size())) + radiation_offset = 0 + nextsw_cday = next_calday call mark_as_initialized('fractional_calendar_days_on_end_of_current_timestep') call mark_as_initialized('fractional_calendar_days_on_end_of_next_timestep') + call mark_as_initialized('number_of_seconds_until_next_shortwave_radiation_timestep') + call mark_as_initialized('next_calendar_day_to_perform_shortwave_radiation_for_surface_models') ! Read CAM namelists. filein = "atm_in" // trim(inst_suffix) @@ -302,6 +306,7 @@ subroutine cam_timestep_init() use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use ccpp_kinds, only: kind_phys use musica_ccpp_dependencies, only: set_initial_musica_concentrations + use radiation_namelist, only: use_rad_uniform_angle, rad_uniform_angle real(kind_phys), pointer :: constituents_array(:,:,:) type(ccpp_constituent_prop_ptr_t), pointer :: constituent_properties(:) @@ -350,6 +355,10 @@ subroutine cam_timestep_init() ! call phys_timestep_init() + ! Call share code to get relevant calendar days + nextsw_cday = get_curr_calday(offset=radiation_offset) + next_calday = get_curr_calday(offset=int(get_step_size())) + end subroutine cam_timestep_init ! !----------------------------------------------------------------------- diff --git a/src/control/runtime_opts.F90 b/src/control/runtime_opts.F90 index 7eb79f32..3ab966d1 100644 --- a/src/control/runtime_opts.F90 +++ b/src/control/runtime_opts.F90 @@ -45,6 +45,7 @@ subroutine read_namelist(nlfilename, single_column, scmlat, scmlon) use inic_analytic_utils, only: analytic_ic_readnl use tropopause_climo_read, only: tropopause_climo_readnl + use radiation_namelist, only: radiation_readnl ! use tracers, only: tracers_readnl ! use nudging, only: nudging_readnl @@ -102,6 +103,7 @@ subroutine read_namelist(nlfilename, single_column, scmlat, scmlon) ! call check_energy_readnl(nlfilename) call analytic_ic_readnl(nlfilename) call tropopause_climo_readnl(nlfilename) + call radiation_readnl(nlfilename) ! call scam_readnl(nlfilename, single_column, scmlat, scmlon) ! call nudging_readnl(nlfilename) diff --git a/src/data/cam_var_init_marks.inc b/src/data/cam_var_init_marks.inc index b9b44c03..f1968b24 100644 --- a/src/data/cam_var_init_marks.inc +++ b/src/data/cam_var_init_marks.inc @@ -67,7 +67,7 @@ end subroutine mark_as_read_from_file - logical function is_initialized(varname) + logical function is_initialized(varname, error_on_not_found) ! This function checks if the variable, , is already ! initialized according to the 'initialized_vars' array. @@ -76,15 +76,23 @@ ! Dummy argument character(len=*), intent(in) :: varname ! Variable name being checked + logical, optional, intent(in) :: error_on_not_found ! Local variables integer :: stdnam_idx ! Standard name array index logical :: found ! Check that was found + logical :: error_on_not_found_loc character(len=*), parameter :: subname = 'is_initialized: ' is_initialized = .false. found = .false. + if (present(error_on_not_found)) then + error_on_not_found_loc = error_on_not_found + else + error_on_not_found_loc = .true. + end if + ! Check if variable is initialized (PARAM, INITIALIZED, or READ_FROM_FILE) do stdnam_idx = 1, phys_var_num if (trim(phys_var_stdnames(stdnam_idx)) == trim(varname)) then @@ -94,7 +102,7 @@ end if end do - if (.not. found) then + if (.not. found .and. error_on_not_found_loc) then ! This condition is an internal error, it should not happen call endrun(subname//": Variable '"//trim(varname)// & "' is missing from phys_var_stdnames array.") diff --git a/src/data/generate_registry_data.py b/src/data/generate_registry_data.py index c2e1cf77..bfea446c 100755 --- a/src/data/generate_registry_data.py +++ b/src/data/generate_registry_data.py @@ -306,7 +306,7 @@ def write_initial_value(self, outfile, indent, init_var, ddt_str, physconst_vars raise TypeError(emsg.format(var_name, self.var_type)) # end if elif init_val: - outfile.write(f"if ({init_var}) then", indent) + outfile.write(f"if ({init_var} .and. .not. is_initialized('{self.standard_name}', error_on_not_found=.false.)) then", indent) outfile.write(f"{var_name} = {init_val}", indent+1) if self.initial_val_vars and self.initial_val_vars.issubset(physconst_vars): outfile.write(f"call mark_as_initialized('{self.standard_name}')", indent+1) diff --git a/src/data/registry.xml b/src/data/registry.xml index 2937643e..c08627ee 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -34,6 +34,7 @@ + @@ -473,6 +474,16 @@ units="1" type="real" kind="kind_phys"> fractional calendar day at end of next timestep + + fractional calendar day at which to next perform shortwave radiation + + + number of seconds until next shortwave radiation timestep + graupel mass mixing ratio with respect to moist air plus all airborne condensates GRAUQM cnst_GRAUQM + + Q cnst_Q + + + CO2 cnst_CO2 + + + N2O cnst_N2O + + + CH4 cnst_CH4 + + + CFC12 cnst_CFC12 + + + CFC11STAR pbuf_CFC11STAR + + + O2 pbuf_O2 + + + ozone pbuf_ozone + + units="count" type="integer"> + -1 + + + -1 + + + -1 + + + -1 + + -1 horizontal_dimension aldif cam_in_aldif + + horizontal_dimension + flwds cam_out_flwds + diff --git a/src/data/registry_v1_0.xsd b/src/data/registry_v1_0.xsd index d6a28461..a4772445 100644 --- a/src/data/registry_v1_0.xsd +++ b/src/data/registry_v1_0.xsd @@ -35,7 +35,7 @@ - + diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index a3df1acd..98470f5c 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit a3df1acde956130a566460604ae65fa9c2219d9c +Subproject commit 98470f5c959d6c6f2765420cc8ea8f7d667766b1 diff --git a/src/physics/utils/radiation_namelist.F90 b/src/physics/utils/radiation_namelist.F90 index f5c75c78..07e67c67 100644 --- a/src/physics/utils/radiation_namelist.F90 +++ b/src/physics/utils/radiation_namelist.F90 @@ -41,8 +41,9 @@ subroutine radiation_readnl(nlfile) use spmd_utils, only: mpicom use cam_logfile, only: iulog use cam_abortutils, only: endrun - use spmd_utils, only: masterproc + use spmd_utils, only: masterproc, masterprocid use time_manager, only: get_step_size + use shr_kind_mod, only: shr_kind_cm !----------------------------------------------------------------------- ! @@ -55,7 +56,7 @@ subroutine radiation_readnl(nlfile) ! Local variables integer :: unitn, ierr - character(len=*), parameter :: subname = 'radiation_readnl' + character(len=*), parameter :: sub = 'radiation_readnl' character(len=shr_kind_cm) :: errmsg integer :: dtime @@ -70,29 +71,29 @@ subroutine radiation_readnl(nlfile) if (ierr == 0) then read(unitn, radiation_nl, iostat=ierr, iomsg=errmsg) if (ierr /= 0) then - call endrun(subname // ':: ERROR reading namelist:' // errmsg) + call endrun(sub // ':: ERROR reading namelist:' // errmsg) end if end if close(unitn) end if ! Broadcast namelist variable - call mpi_bcast(use_rad_uniform_angle, 1, mpi_logical, masterproc, mpicom, ierr) + call mpi_bcast(use_rad_uniform_angle, 1, mpi_logical, masterprocid, mpicom, ierr) if (ierr /= 0) then call endrun(sub//": FATAL: mpi_bcast: use_rad_uniform_angle", & file=__FILE__, line=__LINE__) end if - call mpi_bcast(iradsw, 1, mpi_integer, masterproc, mpicom, ierr) + call mpi_bcast(iradsw, 1, mpi_integer, masterprocid, mpicom, ierr) if (ierr /= 0) then call endrun(sub//": FATAL: mpi_bcast: iradsw", & file=__FILE__, line=__LINE__) end if - call mpi_bcast(iradlw, 1, mpi_integer, masterproc, mpicom, ierr) + call mpi_bcast(iradlw, 1, mpi_integer, masterprocid, mpicom, ierr) if (ierr /= 0) then call endrun(sub//": FATAL: mpi_bcast: iradlw", & file=__FILE__, line=__LINE__) end if - call mpi_bcast(irad_always, 1, mpi_integer, masterproc, mpicom, ierr) + call mpi_bcast(irad_always, 1, mpi_integer, masterprocid, mpicom, ierr) if (ierr /= 0) then call endrun(sub//": FATAL: mpi_bcast: irad_always", & file=__FILE__, line=__LINE__) @@ -103,16 +104,16 @@ subroutine radiation_readnl(nlfile) file=__FILE__, line=__LINE__) end if - if (use_rad_uniform_angle .and. rad_uniform_angle == -99._r8) then + if (use_rad_uniform_angle .and. rad_uniform_angle == -99._kind_phys) then call endrun(sub//': ERROR - use_rad_uniform_angle is set to .true,' & //' but rad_uniform_angle is not set ') end if ! Convert iradsw, iradlw and irad_always from hours to timesteps if necessary dtime = get_step_size() - if (iradsw < 0) iradsw = nint((-iradsw *3600._r8)/dtime) - if (iradlw < 0) iradlw = nint((-iradlw *3600._r8)/dtime) - if (irad_always < 0) irad_always = nint((-irad_always*3600._r8)/dtime) + if (iradsw < 0) iradsw = nint((-iradsw *3600._kind_phys)/dtime) + if (iradlw < 0) iradlw = nint((-iradlw *3600._kind_phys)/dtime) + if (irad_always < 0) irad_always = nint((-irad_always*3600._kind_phys)/dtime) !----------------------------------------------------------------------- ! Print runtime options to log. @@ -120,7 +121,7 @@ subroutine radiation_readnl(nlfile) if (masterproc) then write(iulog,*) 'RRTMGP radiation scheme parameters:' - write(iulog,10) iradsw, iradlw, irad_always, use_rad_dt_cosz + write(iulog,10) iradsw, iradlw, irad_always, use_rad_uniform_angle end if 10 format(' Frequency (timesteps) of Shortwave radiation calc: ',i5/, & diff --git a/src/physics/utils/radiation_namelist.meta b/src/physics/utils/radiation_namelist.meta index 8d68f9b2..2a7e0212 100644 --- a/src/physics/utils/radiation_namelist.meta +++ b/src/physics/utils/radiation_namelist.meta @@ -18,7 +18,7 @@ dimensions = () protected = True [ irad_always ] - standard_name = number_of_timesteps_to_force_radiation_calculation_after_initialization + standard_name = number_of_timesteps_to_force_radiation_calculation_after_initialization_namelist_parameter units = count type = integer dimensions = () diff --git a/src/utils/machine.F90 b/src/utils/machine.F90 new file mode 100644 index 00000000..4d1a37e4 --- /dev/null +++ b/src/utils/machine.F90 @@ -0,0 +1,12 @@ +! This module is the CAM version of the CCPP generated module of the same name +module machine + + use ccpp_kinds, only: kind_phys => kind_phys + + + implicit none + private + + public kind_phys + +end module machine From a694713c240b2f4a91324c0db8d262a3a5ce448c Mon Sep 17 00:00:00 2001 From: peverwhee Date: Fri, 8 Aug 2025 10:56:25 -0600 Subject: [PATCH 008/150] use output variables for physics check --- src/data/write_init_files.py | 40 +++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/data/write_init_files.py b/src/data/write_init_files.py index 54286690..430ef16b 100644 --- a/src/data/write_init_files.py +++ b/src/data/write_init_files.py @@ -33,6 +33,7 @@ 'number_of_mpi_tasks'} # Variable input types _INPUT_TYPES = set(['in', 'inout']) +_OUTPUT_TYPES = set(['out', 'inout']) # Include files to insert in the module preamble _PHYS_VARS_PREAMBLE_INCS = ["cam_var_init_marks_decl.inc"] @@ -131,7 +132,7 @@ def write_init_files(cap_database, ic_names, registry_constituents, vars_init_va # Gather all the host model variables that are required by # any of the compiled CCPP physics suites. - host_vars, constituent_set, retmsg = gather_ccpp_req_vars(cap_database, registry_constituents) + in_vars, out_vars, constituent_set, retmsg = gather_ccpp_req_vars(cap_database, registry_constituents) # Quit now if there are missing variables if retmsg: @@ -174,12 +175,12 @@ def write_init_files(cap_database, ic_names, registry_constituents, vars_init_va # end for # Write public parameters: - retvals = write_ic_params(outfile, host_vars, ic_names, registry_constituents) + retvals = write_ic_params(outfile, in_vars, ic_names, registry_constituents) ic_names, ic_max_len, stdname_max_len = retvals # Write initial condition arrays: write_ic_arrays(outfile, ic_names, ic_max_len, - stdname_max_len, host_vars, registry_constituents) + stdname_max_len, in_vars, registry_constituents) # Add "contains" statement: outfile.end_module_header() @@ -234,18 +235,19 @@ def write_init_files(cap_database, ic_names, registry_constituents, vars_init_va # Grab the host dictionary from the database host_dict = cap_database.host_model_dict() - # Collect imported host variables - host_imports = collect_host_var_imports(host_vars, host_dict, constituent_set) - + # Collect imported host variables for physics read + host_imports = collect_host_var_imports(in_vars, host_dict, constituent_set) # Write physics_read_data subroutine: - write_phys_read_subroutine(outfile, host_dict, host_vars, host_imports, + write_phys_read_subroutine(outfile, host_dict, in_vars, host_imports, phys_check_fname_str, constituent_set, vars_init_value) outfile.blank_line() + # Collect imported host variables for physics check + host_imports = collect_host_var_imports(out_vars, host_dict, constituent_set) # Write physics_check_data subroutine: - write_phys_check_subroutine(outfile, host_dict, host_vars, host_imports, + write_phys_check_subroutine(outfile, host_dict, out_vars, host_imports, phys_check_fname_str, constituent_set) # -------------------------------------- @@ -315,7 +317,8 @@ def gather_ccpp_req_vars(cap_database, registry_constituents): # Dictionary of all 'in' and 'inout' suite variables. # Key is standard name, value is host-model or constituent variable - req_vars = {} + in_vars = {} + out_vars = {} missing_vars = set() constituent_vars = set() retmsg = "" @@ -330,22 +333,31 @@ def gather_ccpp_req_vars(cap_database, registry_constituents): intent = cvar.get_prop_value('intent') is_const = cvar.get_prop_value('advected') or cvar.get_prop_value('constituent') if ((intent in _INPUT_TYPES) and - (stdname not in req_vars) and + (stdname not in in_vars) and (stdname not in _EXCLUDED_STDNAMES)): if is_const: #Add variable to constituent set: constituent_vars.add(stdname) #Add variable to required variable list if it's not a registry constituent if stdname not in registry_constituents: - req_vars[stdname] = cvar + in_vars[stdname] = cvar # end if else: # We need to work with the host model version of this variable missing = _find_and_add_host_variable(stdname, host_dict, - req_vars) + in_vars) missing_vars.update(missing) # end if - # end if (do not include output variables) + # end if (only input variables) + if ((intent in _OUTPUT_TYPES) and + (stdname not in out_vars) and + (stdname not in _EXCLUDED_STDNAMES)): + if not is_const: + missing = _find_and_add_host_variable(stdname, host_dict, + out_vars) + # do nothing with missing variables + # end if + # end if (only output variables) # end for (loop over call list) # end for (loop over phases) @@ -354,7 +366,7 @@ def gather_ccpp_req_vars(cap_database, registry_constituents): retmsg = f"Error: Missing required host variables: {mvlist}" # end if # Return the required variables as a list - return list(req_vars.values()), constituent_vars, retmsg + return list(in_vars.values()), list(out_vars.values()), constituent_vars, retmsg ########################## #FORTRAN WRITING FUNCTIONS From c2a33d7b24ac09b0df50ae12c76a8fe4e64f1e22 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Fri, 8 Aug 2025 11:13:14 -0600 Subject: [PATCH 009/150] fix unit tests --- .../sample_files/write_init_files/physics_inputs_4D.F90 | 5 +---- .../sample_files/write_init_files/physics_inputs_bvd.F90 | 5 +---- .../write_init_files/physics_inputs_cnst.F90 | 6 +----- .../write_init_files/physics_inputs_constituent_dim.F90 | 6 +----- .../sample_files/write_init_files/physics_inputs_ddt.F90 | 6 +----- .../write_init_files/physics_inputs_ddt2.F90 | 4 ---- .../write_init_files/physics_inputs_ddt_array.F90 | 4 ---- .../write_init_files/physics_inputs_host_var.F90 | 5 ----- .../write_init_files/physics_inputs_initial_value.F90 | 6 +----- .../sample_files/write_init_files/physics_inputs_mf.F90 | 5 ----- .../write_init_files/physics_inputs_no_horiz.F90 | 5 +---- .../write_init_files/physics_inputs_param.F90 | 9 +-------- .../write_init_files/physics_inputs_protect.F90 | 6 +----- .../write_init_files/physics_inputs_scalar.F90 | 5 +---- .../write_init_files/physics_inputs_simple.F90 | 6 +++--- .../python/sample_files/write_init_files/simple_reg.xml | 5 +++++ .../python/sample_files/write_init_files/temp_adjust.F90 | 5 ++++- .../sample_files/write_init_files/temp_adjust.meta | 6 ++++++ 18 files changed, 28 insertions(+), 71 deletions(-) diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_4D.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_4D.F90 index 40a1e332..1ffab12e 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_4D.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_4D.F90 @@ -235,7 +235,7 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, use cam_pio_utils, only: cam_pio_openfile, cam_pio_closefile use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use phys_vars_init_check_4D, only: phys_var_num, phys_var_stdnames, input_var_names, std_name_len - use physics_types_4D, only: slp, theta + use physics_types_4D, only: theta ! Dummy arguments character(len=SHR_KIND_CL), intent(in) :: file_name @@ -327,9 +327,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call endrun('Cannot check status of slp'//', slp has unsupported dimension, timestep_for_physics.') - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_bvd.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_bvd.F90 index 9bf56d36..82f38ebd 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_bvd.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_bvd.F90 @@ -235,7 +235,7 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, use cam_pio_utils, only: cam_pio_openfile, cam_pio_closefile use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use phys_vars_init_check_bvd, only: phys_var_num, phys_var_stdnames, input_var_names, std_name_len - use physics_types_bad_vertdim, only: slp, theta + use physics_types_bad_vertdim, only: theta ! Dummy arguments character(len=SHR_KIND_CL), intent(in) :: file_name @@ -327,9 +327,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call endrun('Cannot check status of slp'//', slp has unsupported dimension, band_number.') - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_cnst.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_cnst.F90 index 7de9cb05..c8d16a6f 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_cnst.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_cnst.F90 @@ -235,7 +235,7 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, use cam_pio_utils, only: cam_pio_openfile, cam_pio_closefile use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use phys_vars_init_check_cnst, only: phys_var_num, phys_var_stdnames, input_var_names, std_name_len - use physics_types_simple, only: slp, theta + use physics_types_simple, only: theta ! Dummy arguments character(len=SHR_KIND_CL), intent(in) :: file_name @@ -327,10 +327,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call check_field(file, input_var_names(:,name_idx), timestep, slp, 'air_pressure_at_sea_level', min_difference, min_relative_value, & - is_first, diff_found) - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_constituent_dim.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_constituent_dim.F90 index 9598eb58..edb6f294 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_constituent_dim.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_constituent_dim.F90 @@ -243,7 +243,7 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, use cam_pio_utils, only: cam_pio_openfile, cam_pio_closefile use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use phys_vars_init_check_constituent_dim, only: phys_var_num, phys_var_stdnames, input_var_names, std_name_len - use physics_types_simple_constituent_dim, only: cool_cat_for_each_const, cool_default_cat_for_each_const, slp, theta + use physics_types_simple_constituent_dim, only: cool_cat_for_each_const, theta ! Dummy arguments character(len=SHR_KIND_CL), intent(in) :: file_name @@ -335,10 +335,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call check_field(file, input_var_names(:,name_idx), timestep, slp, 'air_pressure_at_sea_level', min_difference, min_relative_value, & - is_first, diff_found) - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_ddt.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_ddt.F90 index 9e74f73e..b6fbc069 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_ddt.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_ddt.F90 @@ -235,7 +235,7 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, use cam_pio_utils, only: cam_pio_openfile, cam_pio_closefile use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use phys_vars_init_check_ddt, only: phys_var_num, phys_var_stdnames, input_var_names, std_name_len - use physics_types_ddt, only: phys_state, slp + use physics_types_ddt, only: phys_state ! Dummy arguments character(len=SHR_KIND_CL), intent(in) :: file_name @@ -327,10 +327,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, phys_state%theta, 'potential_temperature', min_difference, & min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call check_field(file, input_var_names(:,name_idx), timestep, slp, 'air_pressure_at_sea_level', min_difference, min_relative_value, & - is_first, diff_found) - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_ddt2.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_ddt2.F90 index f8814a30..c4afce3a 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_ddt2.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_ddt2.F90 @@ -327,10 +327,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, phys_state%thermo%theta, 'potential_temperature', & min_difference, min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call check_field(file, input_var_names(:,name_idx), timestep, phys_state%slp, 'air_pressure_at_sea_level', min_difference, & - min_relative_value, is_first, diff_found) - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_ddt_array.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_ddt_array.F90 index c8171a6d..42ce5415 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_ddt_array.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_ddt_array.F90 @@ -327,10 +327,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, phys_state%t(:, :, ix_theta), 'potential_temperature', & min_difference, min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call check_field(file, input_var_names(:,name_idx), timestep, phys_state%slp, 'air_pressure_at_sea_level', min_difference, & - min_relative_value, is_first, diff_found) - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_host_var.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_host_var.F90 index 6e4e750c..2985bbe0 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_host_var.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_host_var.F90 @@ -232,7 +232,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, use cam_pio_utils, only: cam_pio_openfile, cam_pio_closefile use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use phys_vars_init_check_host_var, only: phys_var_num, phys_var_stdnames, input_var_names, std_name_len - use physics_types_host_var, only: slp ! Dummy arguments character(len=SHR_KIND_CL), intent(in) :: file_name @@ -320,10 +319,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, ! Check variable vs input check file: select case (trim(phys_var_stdnames(name_idx))) - case ('air_pressure_at_sea_level') - call check_field(file, input_var_names(:,name_idx), timestep, slp, 'air_pressure_at_sea_level', min_difference, min_relative_value, & - is_first, diff_found) - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_initial_value.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_initial_value.F90 index b5c19c57..a09a6b4c 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_initial_value.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_initial_value.F90 @@ -235,7 +235,7 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, use cam_pio_utils, only: cam_pio_openfile, cam_pio_closefile use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use phys_vars_init_check_initial_value, only: phys_var_num, phys_var_stdnames, input_var_names, std_name_len - use physics_types_simple_initial_value, only: slp, theta + use physics_types_simple_initial_value, only: theta ! Dummy arguments character(len=SHR_KIND_CL), intent(in) :: file_name @@ -327,10 +327,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call check_field(file, input_var_names(:,name_idx), timestep, slp, 'air_pressure_at_sea_level', min_difference, min_relative_value, & - is_first, diff_found) - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_mf.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_mf.F90 index 5b1fbf68..6b07bbba 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_mf.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_mf.F90 @@ -237,7 +237,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use phys_vars_init_check_mf, only: phys_var_num, phys_var_stdnames, input_var_names, std_name_len use ref_theta, only: theta - use physics_types_mf, only: slp ! Dummy arguments character(len=SHR_KIND_CL), intent(in) :: file_name @@ -329,10 +328,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call check_field(file, input_var_names(:,name_idx), timestep, slp, 'air_pressure_at_sea_level', min_difference, min_relative_value, & - is_first, diff_found) - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_no_horiz.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_no_horiz.F90 index ce6db2e5..c35112a9 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_no_horiz.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_no_horiz.F90 @@ -235,7 +235,7 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, use cam_pio_utils, only: cam_pio_openfile, cam_pio_closefile use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use phys_vars_init_check_no_horiz, only: phys_var_num, phys_var_stdnames, input_var_names, std_name_len - use physics_types_no_horiz, only: slp, theta + use physics_types_no_horiz, only: theta ! Dummy arguments character(len=SHR_KIND_CL), intent(in) :: file_name @@ -327,9 +327,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call endrun('Cannot check status of slp'//', slp has no horizontal dimension') - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_param.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_param.F90 index cedc2e65..1460f086 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_param.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_param.F90 @@ -238,7 +238,7 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, use cam_pio_utils, only: cam_pio_openfile, cam_pio_closefile use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use phys_vars_init_check_param, only: phys_var_num, phys_var_stdnames, input_var_names, std_name_len - use physics_types_param, only: g, slp, theta + use physics_types_param, only: theta ! Dummy arguments character(len=SHR_KIND_CL), intent(in) :: file_name @@ -330,13 +330,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call check_field(file, input_var_names(:,name_idx), timestep, slp, 'air_pressure_at_sea_level', min_difference, min_relative_value, & - is_first, diff_found) - - case ('gravitational_acceleration') - call endrun('Cannot check status of g'//', g has no horizontal dimension') - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_protect.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_protect.F90 index c904aad1..52ae86ec 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_protect.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_protect.F90 @@ -235,7 +235,7 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, use cam_pio_utils, only: cam_pio_openfile, cam_pio_closefile use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use phys_vars_init_check_protect, only: phys_var_num, phys_var_stdnames, input_var_names, std_name_len - use physics_types_protected, only: slp, theta + use physics_types_protected, only: theta ! Dummy arguments character(len=SHR_KIND_CL), intent(in) :: file_name @@ -327,10 +327,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call check_field(file, input_var_names(:,name_idx), timestep, slp, 'air_pressure_at_sea_level', min_difference, min_relative_value, & - is_first, diff_found) - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_scalar.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_scalar.F90 index 60214bc5..73836183 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_scalar.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_scalar.F90 @@ -235,7 +235,7 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, use cam_pio_utils, only: cam_pio_openfile, cam_pio_closefile use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use phys_vars_init_check_scalar, only: phys_var_num, phys_var_stdnames, input_var_names, std_name_len - use physics_types_scalar_var, only: slp, theta + use physics_types_scalar_var, only: theta ! Dummy arguments character(len=SHR_KIND_CL), intent(in) :: file_name @@ -327,9 +327,6 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call endrun('Cannot check status of slp'//', slp has no horizontal dimension') - end select !check variables if (diff_found) then overall_diff_found = .true. diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_simple.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_simple.F90 index ae1222cc..3d6ce2b1 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_simple.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_simple.F90 @@ -235,7 +235,7 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, use cam_pio_utils, only: cam_pio_openfile, cam_pio_closefile use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t use phys_vars_init_check_simple, only: phys_var_num, phys_var_stdnames, input_var_names, std_name_len - use physics_types_simple, only: slp, theta + use physics_types_simple, only: ptend, theta ! Dummy arguments character(len=SHR_KIND_CL), intent(in) :: file_name @@ -327,8 +327,8 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & min_relative_value, is_first, diff_found) - case ('air_pressure_at_sea_level') - call check_field(file, input_var_names(:,name_idx), timestep, slp, 'air_pressure_at_sea_level', min_difference, min_relative_value, & + case ('tendency_of_peverwhee') + call check_field(file, input_var_names(:,name_idx), timestep, ptend, 'tendency_of_peverwhee', min_difference, min_relative_value, & is_first, diff_found) end select !check variables diff --git a/test/unit/python/sample_files/write_init_files/simple_reg.xml b/test/unit/python/sample_files/write_init_files/simple_reg.xml index d9b0892d..a5b686b2 100644 --- a/test/unit/python/sample_files/write_init_files/simple_reg.xml +++ b/test/unit/python/sample_files/write_init_files/simple_reg.xml @@ -13,6 +13,11 @@ horizontal_dimension slp sea_lev_pres + + horizontal_dimension + ptend + horizontal_dimension diff --git a/test/unit/python/sample_files/write_init_files/temp_adjust.F90 b/test/unit/python/sample_files/write_init_files/temp_adjust.F90 index 29ce43e9..0bed6887 100644 --- a/test/unit/python/sample_files/write_init_files/temp_adjust.F90 +++ b/test/unit/python/sample_files/write_init_files/temp_adjust.F90 @@ -18,7 +18,7 @@ MODULE temp_adjust !! \htmlinclude arg_table_temp_adjust_run.html !! SUBROUTINE temp_adjust_run(nbox, lev, temp_layer, & - slp, timestep, errmsg, errflg) + slp, timestep, ptend, errmsg, errflg) !---------------------------------------------------------------- IMPLICIT NONE !---------------------------------------------------------------- @@ -27,6 +27,7 @@ SUBROUTINE temp_adjust_run(nbox, lev, temp_layer, & REAL(kind_phys), intent(inout) :: temp_layer(:, :) real(kind_phys), intent(in) :: slp(:) real(kind_phys), intent(in) :: timestep + real(kind_phys), intent(out) :: ptend(:) character(len=512), intent(out) :: errmsg integer, intent(out) :: errflg !---------------------------------------------------------------- @@ -37,6 +38,8 @@ SUBROUTINE temp_adjust_run(nbox, lev, temp_layer, & errmsg = '' errflg = 0 + ptend = 0._kind_phys + do box_index = 1, nbox do lev_index = 1, lev temp_layer(box_index, lev_index) = temp_layer(box_index, lev_index) & diff --git a/test/unit/python/sample_files/write_init_files/temp_adjust.meta b/test/unit/python/sample_files/write_init_files/temp_adjust.meta index d132fdf1..79153832 100644 --- a/test/unit/python/sample_files/write_init_files/temp_adjust.meta +++ b/test/unit/python/sample_files/write_init_files/temp_adjust.meta @@ -38,6 +38,12 @@ type = real kind = kind_phys intent = in +[ ptend ] + standard_name = tendency_of_peverwhee + units = s-1 + dimensions = (horizontal_loop_extent) + type = real | kind = kind_phys + intent = out [ errmsg ] standard_name = ccpp_error_message long_name = Error message for error handling in CCPP From 8b9df0a4a78dc43a2a11d4e4a3f169d83055d2b0 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Fri, 8 Aug 2025 17:03:43 -0600 Subject: [PATCH 010/150] update check_data to not error when variable is not found --- src/data/write_init_files.py | 73 +++++++++++-------- .../phys_vars_init_check_simple.F90 | 10 ++- .../write_init_files/physics_inputs_4D.F90 | 50 +++++++------ .../write_init_files/physics_inputs_bvd.F90 | 50 +++++++------ .../write_init_files/physics_inputs_cnst.F90 | 50 +++++++------ .../physics_inputs_constituent_dim.F90 | 50 +++++++------ .../write_init_files/physics_inputs_ddt.F90 | 50 +++++++------ .../write_init_files/physics_inputs_ddt2.F90 | 50 +++++++------ .../physics_inputs_ddt_array.F90 | 50 +++++++------ .../physics_inputs_host_var.F90 | 44 ++++++----- .../physics_inputs_initial_value.F90 | 50 +++++++------ .../write_init_files/physics_inputs_mf.F90 | 50 +++++++------ .../physics_inputs_no_horiz.F90 | 50 +++++++------ .../write_init_files/physics_inputs_noreq.F90 | 44 ++++++----- .../write_init_files/physics_inputs_param.F90 | 50 +++++++------ .../physics_inputs_protect.F90 | 50 +++++++------ .../physics_inputs_scalar.F90 | 50 +++++++------ .../physics_inputs_simple.F90 | 56 +++++++------- 18 files changed, 495 insertions(+), 382 deletions(-) diff --git a/src/data/write_init_files.py b/src/data/write_init_files.py index 430ef16b..014c70fc 100644 --- a/src/data/write_init_files.py +++ b/src/data/write_init_files.py @@ -174,13 +174,17 @@ def write_init_files(cap_database, ic_names, registry_constituents, vars_init_va outfile.include(filepath) # end for + all_req_vars = in_vars + out_vars + # Remove duplicates but preserve order for testing + unique_req_vars = list(OrderedDict.fromkeys(all_req_vars)) + # Write public parameters: - retvals = write_ic_params(outfile, in_vars, ic_names, registry_constituents) + retvals = write_ic_params(outfile, unique_req_vars, ic_names, registry_constituents) ic_names, ic_max_len, stdname_max_len = retvals # Write initial condition arrays: write_ic_arrays(outfile, ic_names, ic_max_len, - stdname_max_len, in_vars, registry_constituents) + stdname_max_len, unique_req_vars, registry_constituents) # Add "contains" statement: outfile.end_module_header() @@ -1190,8 +1194,6 @@ def write_phys_read_subroutine(outfile, host_dict, host_vars, host_imports, outfile.write("end do", 2) outfile.blank_line() - # start default case steps: - # End subroutine: outfile.write("end subroutine physics_read_data", 1) @@ -1252,8 +1254,8 @@ def write_phys_check_subroutine(outfile, host_dict, host_vars, host_imports, call_str += f"timestep, {var_locname}, '{var_stdname}', " call_str += "min_difference, min_relative_value, is_first, diff_found)" else: - call_str = f"call endrun('Cannot check status of {var_locname}'" + \ - f"//', {reason}')" + # For check field, don't endrun + call_str = "! do nothing - variable can't be checked" # end if # Add string to dictionary: call_string_dict[call_string_key] = call_str @@ -1388,7 +1390,7 @@ def write_phys_check_subroutine(outfile, host_dict, host_vars, host_imports, "that they can be read from input file if need be:", 3) outfile.write("call ccpp_physics_suite_variables(suite_names" + \ "(suite_idx), ccpp_required_data, errmsg, errflg, " + \ - "input_vars=.true., output_vars=.false.)", 4) + "input_vars=.false., output_vars=.true.)", 4) outfile.blank_line() # Loop over required variables: @@ -1397,37 +1399,48 @@ def write_phys_check_subroutine(outfile, host_dict, host_vars, host_imports, outfile.write("do req_idx = 1, size(ccpp_required_data, 1)", 3) outfile.blank_line() - # First check if the required variable is a constituent - outfile.comment("First check if the required variable is a constituent:", 4) - outfile.write("call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.)", 4) - outfile.write("if (constituent_idx > -1) then", 4) - outfile.write("cycle", 5) - outfile.write("else", 4) - outfile.comment("The required variable is not a constituent. Check if the variable was read from a file", 5) - # Call input name search function: - outfile.comment("Find IC file input name array index for required variable:", 5) - outfile.write("call is_read_from_file(ccpp_required_data(req_idx), " + \ - "is_read, stdnam_idx_out=name_idx)", 5) - outfile.write("if (.not. is_read) then", 5) - outfile.write("cycle", 6) - outfile.write("end if", 5) + outfile.comment("Find IC file input name array index for required variable:", 4) + outfile.write("name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx)", 4) + + # Start select-case statement: + outfile.blank_line() + outfile.comment("Check for special index values:", 4) + outfile.write("select case (name_idx)", 4) + outfile.blank_line() + + # Skip constituent variables: + outfile.write("case (const_idx)", 5) + outfile.blank_line() + outfile.comment("If variable is a constituent, then do nothing. We'll handle these later", 6) + outfile.blank_line() + + # Generate error message if required variable isn't found: + outfile.write("case (no_exist_idx)", 5) + outfile.blank_line() + outfile.comment("If an index was never found, then do nothing. We won't try to check these.", 6) + outfile.blank_line() + + # start default case steps: + outfile.write("case default", 5) + outfile.blank_line() # Generate "check_field" calls: - outfile.comment("Check variable vs input check file:", 5) + outfile.comment("Check variable vs input check file:", 6) outfile.blank_line() - outfile.write("select case (trim(phys_var_stdnames(name_idx)))", 5) + outfile.write("select case (trim(phys_var_stdnames(name_idx)))", 6) for case_call, read_call in call_string_dict.items(): - outfile.write(case_call, 5) - outfile.write(read_call, 6) + outfile.write(case_call, 6) + outfile.write(read_call, 7) outfile.blank_line() - outfile.write("end select !check variables", 5) - outfile.write("if (diff_found) then", 5) - outfile.write("overall_diff_found = .true.", 6) - outfile.write("end if", 5) - outfile.write("end if !check if constituent", 4) + outfile.write("end select !check variables", 6) + outfile.write("if (diff_found) then", 6) + outfile.write("overall_diff_found = .true.", 7) + outfile.write("end if", 6) # End select case and required variables loop: + outfile.write("end select !special indices", 4) + outfile.blank_line() outfile.write("end do !Suite-required variables", 3) outfile.blank_line() diff --git a/test/unit/python/sample_files/write_init_files/phys_vars_init_check_simple.F90 b/test/unit/python/sample_files/write_init_files/phys_vars_init_check_simple.F90 index 66f4d984..d9859dd3 100644 --- a/test/unit/python/sample_files/write_init_files/phys_vars_init_check_simple.F90 +++ b/test/unit/python/sample_files/write_init_files/phys_vars_init_check_simple.F90 @@ -33,7 +33,7 @@ module phys_vars_init_check_simple integer, public, parameter :: PARAM = 2 integer, public, parameter :: READ_FROM_FILE = 3 ! Total number of physics-related variables: - integer, public, parameter :: phys_var_num = 2 + integer, public, parameter :: phys_var_num = 3 integer, public, parameter :: phys_const_num = 16 !Max length of physics-related variable standard names: @@ -45,7 +45,8 @@ module phys_vars_init_check_simple ! Physics-related input variable standard names: character(len=25), public, protected :: phys_var_stdnames(phys_var_num) = (/ & 'potential_temperature ', & - 'air_pressure_at_sea_level' /) + 'air_pressure_at_sea_level', & + 'tendency_of_peverwhee ' /) character(len=36), public, protected :: phys_const_stdnames(phys_const_num) = (/ & "ccpp_constituent_minimum_values ", & @@ -67,15 +68,18 @@ module phys_vars_init_check_simple !Array storing all registered IC file input names for each variable: character(len=5), public, protected :: input_var_names(1, phys_var_num) = reshape((/ & 'theta', & - 'slp ' /), (/1, phys_var_num/)) + 'slp ', & + 'ptend' /), (/1, phys_var_num/)) ! Array indicating whether or not variable is protected: logical, public, protected :: protected_vars(phys_var_num)= (/ & + .false., & .false., & .false. /) ! Variable state (UNINITIALIZED, INTIIALIZED, PARAM or READ_FROM_FILE): integer, public, protected :: initialized_vars(phys_var_num)= (/ & + UNINITIALIZED, & UNINITIALIZED, & UNINITIALIZED /) diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_4D.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_4D.F90 index 1ffab12e..65f32f80 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_4D.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_4D.F90 @@ -304,34 +304,40 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & - min_relative_value, is_first, diff_found) + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & + min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_bvd.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_bvd.F90 index 82f38ebd..03a3fb25 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_bvd.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_bvd.F90 @@ -304,34 +304,40 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & - min_relative_value, is_first, diff_found) + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & + min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_cnst.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_cnst.F90 index c8d16a6f..63eb78cc 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_cnst.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_cnst.F90 @@ -304,34 +304,40 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & - min_relative_value, is_first, diff_found) + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & + min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_constituent_dim.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_constituent_dim.F90 index edb6f294..84a8298f 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_constituent_dim.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_constituent_dim.F90 @@ -312,34 +312,40 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & - min_relative_value, is_first, diff_found) + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & + min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_ddt.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_ddt.F90 index b6fbc069..cf58fff7 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_ddt.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_ddt.F90 @@ -304,34 +304,40 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, phys_state%theta, 'potential_temperature', min_difference, & - min_relative_value, is_first, diff_found) + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, phys_state%theta, 'potential_temperature', min_difference, & + min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_ddt2.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_ddt2.F90 index c4afce3a..f05e2cc0 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_ddt2.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_ddt2.F90 @@ -304,34 +304,40 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, phys_state%thermo%theta, 'potential_temperature', & - min_difference, min_relative_value, is_first, diff_found) + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, phys_state%thermo%theta, 'potential_temperature', & + min_difference, min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_ddt_array.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_ddt_array.F90 index 42ce5415..6a0aab72 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_ddt_array.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_ddt_array.F90 @@ -304,34 +304,40 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, phys_state%t(:, :, ix_theta), 'potential_temperature', & - min_difference, min_relative_value, is_first, diff_found) + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, phys_state%t(:, :, ix_theta), 'potential_temperature', & + min_difference, min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_host_var.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_host_var.F90 index 2985bbe0..8244f7ce 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_host_var.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_host_var.F90 @@ -300,30 +300,36 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) + + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - select case (trim(phys_var_stdnames(name_idx))) - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_initial_value.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_initial_value.F90 index a09a6b4c..9a63c37f 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_initial_value.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_initial_value.F90 @@ -304,34 +304,40 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & - min_relative_value, is_first, diff_found) + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & + min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_mf.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_mf.F90 index 6b07bbba..7398ac66 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_mf.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_mf.F90 @@ -305,34 +305,40 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & - min_relative_value, is_first, diff_found) + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & + min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_no_horiz.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_no_horiz.F90 index c35112a9..c6b03016 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_no_horiz.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_no_horiz.F90 @@ -304,34 +304,40 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & - min_relative_value, is_first, diff_found) + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & + min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_noreq.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_noreq.F90 index 3d655e72..b814b2e7 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_noreq.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_noreq.F90 @@ -296,30 +296,36 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) + + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - select case (trim(phys_var_stdnames(name_idx))) - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_param.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_param.F90 index 1460f086..3b4170f5 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_param.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_param.F90 @@ -307,34 +307,40 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & - min_relative_value, is_first, diff_found) + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & + min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_protect.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_protect.F90 index 52ae86ec..f2b8defd 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_protect.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_protect.F90 @@ -304,34 +304,40 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & - min_relative_value, is_first, diff_found) + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & + min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_scalar.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_scalar.F90 index 73836183..d012f814 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_scalar.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_scalar.F90 @@ -304,34 +304,40 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & - min_relative_value, is_first, diff_found) + case (const_idx) + + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & + min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: diff --git a/test/unit/python/sample_files/write_init_files/physics_inputs_simple.F90 b/test/unit/python/sample_files/write_init_files/physics_inputs_simple.F90 index 3d6ce2b1..40f54c21 100644 --- a/test/unit/python/sample_files/write_init_files/physics_inputs_simple.F90 +++ b/test/unit/python/sample_files/write_init_files/physics_inputs_simple.F90 @@ -304,38 +304,44 @@ subroutine physics_check_data(file_name, suite_names, timestep, min_difference, do suite_idx = 1, size(suite_names, 1) ! Search for all needed CCPP input variables, so that they can be read from input file if need be: - call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.true., output_vars=.false.) + call ccpp_physics_suite_variables(suite_names(suite_idx), ccpp_required_data, errmsg, errflg, input_vars=.false., output_vars=.true.) ! Loop over all required variables as specified by CCPP suite: do req_idx = 1, size(ccpp_required_data, 1) - ! First check if the required variable is a constituent: - call const_get_index(ccpp_required_data(req_idx), constituent_idx, abort=.false., warning=.false.) - if (constituent_idx > -1) then - cycle - else - ! The required variable is not a constituent. Check if the variable was read from a file - ! Find IC file input name array index for required variable: - call is_read_from_file(ccpp_required_data(req_idx), is_read, stdnam_idx_out=name_idx) - if (.not. is_read) then - cycle - end if - ! Check variable vs input check file: + ! Find IC file input name array index for required variable: + name_idx = find_input_name_idx(ccpp_required_data(req_idx), .true., constituent_idx) + + ! Check for special index values: + select case (name_idx) - select case (trim(phys_var_stdnames(name_idx))) - case ('potential_temperature') - call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & - min_relative_value, is_first, diff_found) + case (const_idx) - case ('tendency_of_peverwhee') - call check_field(file, input_var_names(:,name_idx), timestep, ptend, 'tendency_of_peverwhee', min_difference, min_relative_value, & - is_first, diff_found) + ! If variable is a constituent, then do nothing. We'll handle these later + + case (no_exist_idx) + + ! If an index was never found, then do nothing. We won't try to check these. + + case default + + ! Check variable vs input check file: + + select case (trim(phys_var_stdnames(name_idx))) + case ('potential_temperature') + call check_field(file, input_var_names(:,name_idx), 'lev', timestep, theta, 'potential_temperature', min_difference, & + min_relative_value, is_first, diff_found) + + case ('tendency_of_peverwhee') + call check_field(file, input_var_names(:,name_idx), timestep, ptend, 'tendency_of_peverwhee', min_difference, & + min_relative_value, is_first, diff_found) + + end select !check variables + if (diff_found) then + overall_diff_found = .true. + end if + end select !special indices - end select !check variables - if (diff_found) then - overall_diff_found = .true. - end if - end if !check if constituent end do !Suite-required variables ! Deallocate required variables array for use in next suite: From 275203082ee45e86c060d8d289ed337e97e0d1d8 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Sat, 9 Aug 2025 00:31:21 -0600 Subject: [PATCH 011/150] update atmos phys submodule --- .gitmodules | 2 +- src/physics/ncar_ccpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 5aef8df7..ac8308bd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = be188731ed9fdd59e225117f59910c80f2d2ea1d + fxtag = 18ee2e110b2d970a12b6d67ad37874ad985d10c1 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index be188731..18ee2e11 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit be188731ed9fdd59e225117f59910c80f2d2ea1d +Subproject commit 18ee2e110b2d970a12b6d67ad37874ad985d10c1 From 586a05f2d2798d18cd3833e497c8a4a348cacfd0 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 5 Aug 2025 15:43:31 -0600 Subject: [PATCH 012/150] Add 'var_subset_check' helper routine, with initial 'both present' check and related unit tests. --- .github/workflows/fortran_unit_tests.yml | 1 + src/physics/utils/pio_reader.F90 | 97 ++++++++++++++-- .../fortran/src/pio_reader/test_pio_reader.pf | 106 ++++++++++++++++++ 3 files changed, 194 insertions(+), 10 deletions(-) diff --git a/.github/workflows/fortran_unit_tests.yml b/.github/workflows/fortran_unit_tests.yml index c2143ac7..064c7cce 100644 --- a/.github/workflows/fortran_unit_tests.yml +++ b/.github/workflows/fortran_unit_tests.yml @@ -5,6 +5,7 @@ on: branches: - development - main + - pio_reader_subset pull_request: workflow_dispatch: diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 3868e8c3..aa5f6604 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -9,16 +9,17 @@ module pio_reader public :: pio_reader_t !Error code parameters - integer, parameter :: pio_inq_dim_id_err = 1 - integer, parameter :: pio_inq_dim_len_err = 2 - integer, parameter :: pio_inq_var_id_err = 3 - integer, parameter :: pio_inq_var_info_err = 4 - integer, parameter :: bad_var_rank_err = 5 - integer, parameter :: too_high_rank_err = 6 - integer, parameter :: pio_get_var_err = 7 - integer, parameter :: not_char_type_err = 8 - integer, parameter :: file_not_open_err = 9 - integer, parameter :: pio_get_msg_err = 10 + integer, parameter :: pio_inq_dim_id_err = 1 + integer, parameter :: pio_inq_dim_len_err = 2 + integer, parameter :: pio_inq_var_id_err = 3 + integer, parameter :: pio_inq_var_info_err = 4 + integer, parameter :: bad_var_rank_err = 5 + integer, parameter :: too_high_rank_err = 6 + integer, parameter :: pio_get_var_err = 7 + integer, parameter :: not_char_type_err = 8 + integer, parameter :: file_not_open_err = 9 + integer, parameter :: pio_get_msg_err = 10 + integer, parameter :: mismatch_start_count_err = 11 type :: file_handle_t logical :: is_file_open = .false. !Is NetCDF file currently open? @@ -297,6 +298,14 @@ subroutine get_netcdf_var_int_1d(this, varname, var, errmsg, errcode, start, cou return end if + !Check if variable subsetting is requested and valid: + call var_subset_check(var_ndims, dim_sizes, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and !read-in the NetCDF data: allocate(var(dim_sizes(1)), stat=errcode, errmsg=errmsg) @@ -2152,4 +2161,72 @@ subroutine get_dim_sizes(varname, var_rank, var_id, pio_file_handle, dim_sizes, end subroutine get_dim_sizes + subroutine var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, start, count) + !Check that the start and count arrays are + !either both absent or present, have the correct number + !of elements, have elements that are within bounds, + !and have element values that work with expected + !number of output variable dimensions. + + !Input arguments: + character(len=*), intent(in) :: varname !Name of NetCDF variable being subset + integer, intent(in) :: var_ndims !Number of output variable dimensions + integer, intent(in) :: dim_sizes(:) !Dimension sizes for variable on file + + !Output arguments: + character(len=*), intent(out) :: errmsg !Error message + integer, intent(out) :: errcode !Error code + + !Optional input arguments: + integer, optional, intent(in) :: start(:) !Start indices for each dimension + integer, optional, intent(in) :: count(:) !Number of elements to read for each dimension + + integer :: i !Loop control variable + + !Initialize error code and message: + errcode = 0 + errmsg = '' + + !If both start and count are not present, + !then return with no error: + if (.not. present(start) .and. .not. present(count)) then + return + end if + + !At this point, either start or count + !must be present, so check that both + !are present: + if (.not. present(start)) then + errcode = mismatch_start_count_err + errmsg = "Variable '"//varname//"' is being subsetted with 'count', but no 'start' was provided." + return + else if (.not. present(count)) then + errcode = mismatch_start_count_err + errmsg = "Variable '"//varname//"' is being subsetted with 'start', but no 'count' was provided." + return + end if + + !Check that start and count are the correct size: + if (size(start) /= var_ndims .or. size(count) /= var_ndims) then + errcode = bad_subset_size_err + errmsg = "Start and count arrays must be the same size as the number of variable dimensions." + return + end if + + !Check that start indices are within bounds: + !do i = 1, var_ndims + ! if (start(i) < 0 .or. start(i) >= dim_sizes(i)) then + ! errcode = bad_start_index_err + ! errmsg = "Start index for dimension "//trim(adjustl(i))//" is out of bounds." + ! return + ! end if + + ! if (count(i) < 1 .or. start(i) + count(i) > dim_sizes(i)) then + ! errcode = bad_count_index_err + ! errmsg = "Count index for dimension "//trim(adjustl(i))//" is out of bounds." + ! return + ! end if + !end do + end subroutine var_subset_check + end module pio_reader diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index feaac718..bdcd8c10 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1247,6 +1247,112 @@ subroutine test_pio_reader_real_get_var_bad_rank() end subroutine test_pio_reader_real_get_var_bad_rank +!+++++++++++++++++++++++++++++++++++++++++++++++++++++ +!PIO reader variable subsetting tests +!+++++++++++++++++++++++++++++++++++++++++++++++++++++ + +@test +subroutine test_pio_reader_missing_start_argument() + ! Check that the PIO reader raises the correct error + ! when attempting to read a variable with + ! a "count" argument provided, but with no "start" + ! argument. + + use funit + + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t + + use ccpp_kinds, only: kind_phys + + class(abstract_netcdf_reader_t), pointer :: reader + + integer :: errcode + character(len=256) :: errmsg + + ! Variable to be read: + integer, allocatable :: pressure(:) + + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + + ! Expected error message: + character(len=*), parameter :: expected_err_msg = & + "Variable 'pressure' is being subsetted with 'count', but no 'start' was provided." + + ! Begin test: + + reader => create_netcdf_reader_t() + + ! Open file: + call reader%open_file(fname, errmsg, errcode) + + ! Attempt to read variable with bad rank: + call reader%get_var("pressure", pressure, errmsg, errcode, count=(/2/)) + + ! Check that the correct error was raised: + @assertNotEqual(0, errcode) + @assertEqual(expected_err_msg, trim(errmsg)) + + ! Perform test cleanup: + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) + +end subroutine test_pio_reader_missing_start_argument + +!----------------------------------------- + +@test +subroutine test_pio_reader_missing_count_argument() + ! Check that the PIO reader raises the correct error + ! when attempting to read a variable with + ! a "start" argument provided, but with no "count" + ! argument. + + use funit + + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t + + use ccpp_kinds, only: kind_phys + + class(abstract_netcdf_reader_t), pointer :: reader + + integer :: errcode + character(len=256) :: errmsg + + ! Variable to be read: + integer, allocatable :: pressure(:) + + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + + ! Expected error message: + character(len=*), parameter :: expected_err_msg = & + "Variable 'pressure' is being subsetted with 'start', but no 'count' was provided." + + ! Begin test: + + reader => create_netcdf_reader_t() + + ! Open file: + call reader%open_file(fname, errmsg, errcode) + + ! Attempt to read variable with bad rank: + call reader%get_var("pressure", pressure, errmsg, errcode, start=(/2/)) + + ! Check that the correct error was raised: + @assertNotEqual(0, errcode) + @assertEqual(expected_err_msg, trim(errmsg)) + + ! Perform test cleanup: + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) + +end subroutine test_pio_reader_missing_count_argument + !++++++++++++++++++++++++++++++++++++++++++++++++++++ @test From d7b4bbe4b744a94e17b0b6f810b79176a57b6b2f Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 5 Aug 2025 15:54:07 -0600 Subject: [PATCH 013/150] Add missing subroutine arguments. --- src/physics/utils/pio_reader.F90 | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index aa5f6604..151c80f5 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -299,7 +299,7 @@ subroutine get_netcdf_var_int_1d(this, varname, var, errmsg, errcode, start, cou end if !Check if variable subsetting is requested and valid: - call var_subset_check(var_ndims, dim_sizes, errmsg, errcode, start, count) + call var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, start, count) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -2169,13 +2169,13 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, star !number of output variable dimensions. !Input arguments: - character(len=*), intent(in) :: varname !Name of NetCDF variable being subset - integer, intent(in) :: var_ndims !Number of output variable dimensions - integer, intent(in) :: dim_sizes(:) !Dimension sizes for variable on file + character(len=*), intent(in) :: varname !Name of NetCDF variable being subset + integer, intent(in) :: var_ndims !Number of output variable dimensions + integer, intent(in) :: dim_sizes(:) !Dimension sizes for variable on file !Output arguments: character(len=*), intent(out) :: errmsg !Error message - integer, intent(out) :: errcode !Error code + integer, intent(out) :: errcode !Error code !Optional input arguments: integer, optional, intent(in) :: start(:) !Start indices for each dimension @@ -2206,12 +2206,14 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, star return end if + !CONTINUE HERE!!!! + !Check that start and count are the correct size: - if (size(start) /= var_ndims .or. size(count) /= var_ndims) then - errcode = bad_subset_size_err - errmsg = "Start and count arrays must be the same size as the number of variable dimensions." - return - end if + !if (size(start) /= var_ndims .or. size(count) /= var_ndims) then + ! errcode = bad_subset_size_err + ! errmsg = "Start and count arrays must be the same size as the number of variable dimensions." + ! return + !end if !Check that start indices are within bounds: !do i = 1, var_ndims From 63a5cbb4ecb0a5f82195b44035ccb86a07c304c3 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 6 Aug 2025 11:31:31 -0600 Subject: [PATCH 014/150] Add check for correct 'start' size, along with new unit test. --- src/physics/utils/pio_reader.F90 | 13 ++++- .../fortran/src/pio_reader/test_pio_reader.pf | 52 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 151c80f5..aa27b8c8 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -19,7 +19,10 @@ module pio_reader integer, parameter :: not_char_type_err = 8 integer, parameter :: file_not_open_err = 9 integer, parameter :: pio_get_msg_err = 10 + + !Subsetting error codes integer, parameter :: mismatch_start_count_err = 11 + integer, parameter :: bad_subset_num_elem_err = 12 type :: file_handle_t logical :: is_file_open = .false. !Is NetCDF file currently open? @@ -2206,7 +2209,15 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, star return end if - !CONTINUE HERE!!!! + !Check that start and count have the correct number of elements: + if (size(start) /= size(dim_sizes)) then + errcode = bad_subset_num_elem_err + write(errmsg, '(3a,i0,a,i0,a)') "The number of elements in the 'start' array for variable '", & + trim(varname), "' does not match the number of dimensions. Expected ", & + size(dim_sizes), " elements, but got", size(start), "." + return + end if + !Check that start and count are the correct size: !if (size(start) /= var_ndims .or. size(count) /= var_ndims) then diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index bdcd8c10..c92aee9d 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1353,6 +1353,58 @@ subroutine test_pio_reader_missing_count_argument() end subroutine test_pio_reader_missing_count_argument +!----------------------------------------- + +@test +subroutine test_pio_reader_bad_num_start_elements() + ! Check that the PIO reader raises the correct error + ! when attempting to read a variable with + ! a "start" argument that has an incorrect number of elements. + + use funit + + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t + + use ccpp_kinds, only: kind_phys + + class(abstract_netcdf_reader_t), pointer :: reader + + integer :: errcode + character(len=256) :: errmsg + + ! Variable to be read: + integer, allocatable :: pressure(:) + + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + + ! Expected error message: + character(len=*), parameter :: expected_err_msg = & + "The number of elements in the 'start' array for variable 'pressure' does not match the number of dimensions. Expected 1 elements, but got 2." + + ! Begin test: + + reader => create_netcdf_reader_t() + + ! Open file: + call reader%open_file(fname, errmsg, errcode) + + ! Attempt to read variable with bad rank: + call reader%get_var("pressure", pressure, errmsg, errcode, start=(/1, 1/)) + + ! Check that the correct error was raised: + @assertNotEqual(0, errcode) + @assertEqual(expected_err_msg, trim(errmsg)) + + ! Perform test cleanup: + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) + +end subroutine test_pio_reader_bad_num_start_elements + + !++++++++++++++++++++++++++++++++++++++++++++++++++++ @test From de58bead0ca7523a9220916d212479a18c8008f9 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 6 Aug 2025 11:36:40 -0600 Subject: [PATCH 015/150] Add missing 'count' entry to test. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index c92aee9d..79658201 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1391,7 +1391,7 @@ subroutine test_pio_reader_bad_num_start_elements() call reader%open_file(fname, errmsg, errcode) ! Attempt to read variable with bad rank: - call reader%get_var("pressure", pressure, errmsg, errcode, start=(/1, 1/)) + call reader%get_var("pressure", pressure, errmsg, errcode, start=(/1, 1/), count=(/2/)) ! Check that the correct error was raised: @assertNotEqual(0, errcode) From eaf9264ef658f6bceb7ac0c4c62929cf7c11ca98 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 6 Aug 2025 14:02:48 -0600 Subject: [PATCH 016/150] Add missing space to error message, a 'count' size check, and an associated unit test. --- src/physics/utils/pio_reader.F90 | 12 ++++- .../fortran/src/pio_reader/test_pio_reader.pf | 50 +++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index aa27b8c8..6e74ec0d 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -2209,15 +2209,23 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, star return end if - !Check that start and count have the correct number of elements: + !Check that start has the correct number of elements: if (size(start) /= size(dim_sizes)) then errcode = bad_subset_num_elem_err write(errmsg, '(3a,i0,a,i0,a)') "The number of elements in the 'start' array for variable '", & trim(varname), "' does not match the number of dimensions. Expected ", & - size(dim_sizes), " elements, but got", size(start), "." + size(dim_sizes), " elements, but got ", size(start), "." return end if + !Check that count has the correct number of elements: + if (size(count) /= size(dim_sizes)) then + errcode = bad_subset_num_elem_err + write(errmsg, '(3a,i0,a,i0,a)') "The number of elements in the 'count' array for variable '", & + trim(varname), "' does not match the number of dimensions. Expected ", & + size(dim_sizes), " elements, but got ", size(count), "." + return + end if !Check that start and count are the correct size: !if (size(start) /= var_ndims .or. size(count) /= var_ndims) then diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 79658201..df888d83 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1404,6 +1404,56 @@ subroutine test_pio_reader_bad_num_start_elements() end subroutine test_pio_reader_bad_num_start_elements +!----------------------------------------- + +@test +subroutine test_pio_reader_bad_num_count_elements() + ! Check that the PIO reader raises the correct error + ! when attempting to read a variable with + ! a "count" argument that has an incorrect number of elements. + + use funit + + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t + + use ccpp_kinds, only: kind_phys + + class(abstract_netcdf_reader_t), pointer :: reader + + integer :: errcode + character(len=256) :: errmsg + + ! Variable to be read: + integer, allocatable :: pressure(:) + + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + + ! Expected error message: + character(len=*), parameter :: expected_err_msg = & + "The number of elements in the 'count' array for variable 'pressure' does not match the number of dimensions. Expected 1 elements, but got 3." + + ! Begin test: + + reader => create_netcdf_reader_t() + + ! Open file: + call reader%open_file(fname, errmsg, errcode) + + ! Attempt to read variable with bad rank: + call reader%get_var("pressure", pressure, errmsg, errcode, start=(/1/), count=(/1,1,1/)) + + ! Check that the correct error was raised: + @assertNotEqual(0, errcode) + @assertEqual(expected_err_msg, trim(errmsg)) + + ! Perform test cleanup: + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) + +end subroutine test_pio_reader_bad_num_count_elements !++++++++++++++++++++++++++++++++++++++++++++++++++++ From f01ec4d16a840c766f9d1f1ec78e91db9338c9d3 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 8 Aug 2025 09:32:17 -0600 Subject: [PATCH 017/150] Add check and test for ensuring that start elements are within range. --- src/physics/utils/pio_reader.F90 | 55 +++++++-------- .../fortran/src/pio_reader/test_pio_reader.pf | 69 +++++++++++++++++-- 2 files changed, 91 insertions(+), 33 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 6e74ec0d..3c5a64e5 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -23,6 +23,7 @@ module pio_reader !Subsetting error codes integer, parameter :: mismatch_start_count_err = 11 integer, parameter :: bad_subset_num_elem_err = 12 + integer, parameter :: bad_subset_range_err = 13 type :: file_handle_t logical :: is_file_open = .false. !Is NetCDF file currently open? @@ -2184,7 +2185,9 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, star integer, optional, intent(in) :: start(:) !Start indices for each dimension integer, optional, intent(in) :: count(:) !Number of elements to read for each dimension - integer :: i !Loop control variable + !Local variables: + integer :: file_var_dim_num !Number of dimensions for variable on file + integer :: i !Loop control variable !Initialize error code and message: errcode = 0 @@ -2210,44 +2213,38 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, star end if !Check that start has the correct number of elements: - if (size(start) /= size(dim_sizes)) then + file_var_dim_num = size(dim_sizes) + if (size(start) /= file_var_dim_num) then errcode = bad_subset_num_elem_err - write(errmsg, '(3a,i0,a,i0,a)') "The number of elements in the 'start' array for variable '", & - trim(varname), "' does not match the number of dimensions. Expected ", & - size(dim_sizes), " elements, but got ", size(start), "." + write(errmsg, '(3a,i0,a,i0,a)') & + "The number of elements in the 'start' array for variable '", & + trim(varname), "' does not match the number of dimensions. Expected ", & + file_var_dim_num, " elements, but got ", size(start), "." return end if !Check that count has the correct number of elements: - if (size(count) /= size(dim_sizes)) then + if (size(count) /= file_var_dim_num) then errcode = bad_subset_num_elem_err - write(errmsg, '(3a,i0,a,i0,a)') "The number of elements in the 'count' array for variable '", & - trim(varname), "' does not match the number of dimensions. Expected ", & - size(dim_sizes), " elements, but got ", size(count), "." + write(errmsg, '(3a,i0,a,i0,a)') & + "The number of elements in the 'count' array for variable '", & + trim(varname), "' does not match the number of dimensions. Expected ", & + file_var_dim_num, " elements, but got ", size(count), "." return end if - !Check that start and count are the correct size: - !if (size(start) /= var_ndims .or. size(count) /= var_ndims) then - ! errcode = bad_subset_size_err - ! errmsg = "Start and count arrays must be the same size as the number of variable dimensions." - ! return - !end if - !Check that start indices are within bounds: - !do i = 1, var_ndims - ! if (start(i) < 0 .or. start(i) >= dim_sizes(i)) then - ! errcode = bad_start_index_err - ! errmsg = "Start index for dimension "//trim(adjustl(i))//" is out of bounds." - ! return - ! end if - - ! if (count(i) < 1 .or. start(i) + count(i) > dim_sizes(i)) then - ! errcode = bad_count_index_err - ! errmsg = "Count index for dimension "//trim(adjustl(i))//" is out of bounds." - ! return - ! end if - !end do + do i = 1, file_var_dim_num + if (start(i) < 0 .or. start(i) > dim_sizes(i)-1) then + errcode = bad_subset_range_err + write(errmsg, '(a,i0,3a,i0,a,i0)') & + "Element ", i, " of 'start' for variable '", & + trim(varname), "' is out of bounds. Expected 0 to ", & + dim_sizes(i), " but got ", start(i), "." + return + end if + end do + end subroutine var_subset_check end module pio_reader diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index df888d83..67991697 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1287,7 +1287,7 @@ subroutine test_pio_reader_missing_start_argument() ! Open file: call reader%open_file(fname, errmsg, errcode) - ! Attempt to read variable with bad rank: + ! Attempt to read variable with missing start argument: call reader%get_var("pressure", pressure, errmsg, errcode, count=(/2/)) ! Check that the correct error was raised: @@ -1339,7 +1339,7 @@ subroutine test_pio_reader_missing_count_argument() ! Open file: call reader%open_file(fname, errmsg, errcode) - ! Attempt to read variable with bad rank: + ! Attempt to read variable with missing count argument: call reader%get_var("pressure", pressure, errmsg, errcode, start=(/2/)) ! Check that the correct error was raised: @@ -1390,7 +1390,7 @@ subroutine test_pio_reader_bad_num_start_elements() ! Open file: call reader%open_file(fname, errmsg, errcode) - ! Attempt to read variable with bad rank: + ! Attempt to read variable with incorrect number of start elements: call reader%get_var("pressure", pressure, errmsg, errcode, start=(/1, 1/), count=(/2/)) ! Check that the correct error was raised: @@ -1441,7 +1441,7 @@ subroutine test_pio_reader_bad_num_count_elements() ! Open file: call reader%open_file(fname, errmsg, errcode) - ! Attempt to read variable with bad rank: + ! Attempt to read variable with incorrect number of count elements: call reader%get_var("pressure", pressure, errmsg, errcode, start=(/1/), count=(/1,1,1/)) ! Check that the correct error was raised: @@ -1455,6 +1455,67 @@ subroutine test_pio_reader_bad_num_count_elements() end subroutine test_pio_reader_bad_num_count_elements +!----------------------------------------- + +@test +subroutine test_pio_reader_bad_start_element_range() + ! Check that the PIO reader raises the correct error + ! when attempting to read a variable with + ! a "start" argument that has an element out of range. + + use funit + + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t + + use ccpp_kinds, only: kind_phys + + class(abstract_netcdf_reader_t), pointer :: reader + + integer :: errcode + character(len=256) :: errmsg + + ! Variable to be read: + integer, allocatable :: pressure(:) + + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + + ! Expected error messages: + character(len=*), parameter :: expected_err_msg_neg = & + "Element 1 of 'start' for variable 'pressure' is out of bounds. Expected 0 to 58 but got -1." + + character(len=*), parameter :: expected_err_msg_huge = & + "Element 1 of 'start' for variable 'pressure' is out of bounds. Expected 0 to 58 but got 200." + + ! Begin test: + + reader => create_netcdf_reader_t() + + ! Open file: + call reader%open_file(fname, errmsg, errcode) + + ! Attempt to read variable with negative start element: + call reader%get_var("pressure", pressure, errmsg, errcode, start=(/-1/), count=(/1/)) + + ! Check that the correct error was raised: + @assertNotEqual(0, errcode) + @assertEqual(expected_err_msg_neg, trim(errmsg)) + + ! Next attempt to read variable with too-large start element: + call reader%get_var("pressure", pressure, errmsg, errcode, start=(/200/), count=(/1/)) + + ! Check that the correct error was raised: + @assertNotEqual(0, errcode) + @assertEqual(expected_err_msg_huge, trim(errmsg)) + + ! Perform test cleanup: + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) + +end subroutine test_pio_reader_bad_start_element_range + !++++++++++++++++++++++++++++++++++++++++++++++++++++ @test From 098f4a0e6f28bde8a6c6a1fdf0e1ab6ecdbb8b3d Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 8 Aug 2025 09:46:00 -0600 Subject: [PATCH 018/150] Add missing Fortran character format. --- src/physics/utils/pio_reader.F90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 3c5a64e5..c883f11e 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -2235,12 +2235,12 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, star !Check that start indices are within bounds: do i = 1, file_var_dim_num - if (start(i) < 0 .or. start(i) > dim_sizes(i)-1) then + if (start(i) < 0 .or. start(i) >= dim_sizes(i)) then errcode = bad_subset_range_err - write(errmsg, '(a,i0,3a,i0,a,i0)') & + write(errmsg, '(a,i0,3a,i0,a,i0,a)') & "Element ", i, " of 'start' for variable '", & trim(varname), "' is out of bounds. Expected 0 to ", & - dim_sizes(i), " but got ", start(i), "." + dim_sizes(i)-1, " but got ", start(i), "." return end if end do From bd07e62f31523bd4488c46a0a8b167473bc33336 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 8 Aug 2025 12:00:40 -0600 Subject: [PATCH 019/150] Add 'alloc_dims' output variable to ensure output variable is allocated to the correct size. --- src/physics/utils/pio_reader.F90 | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index c883f11e..4a1ef452 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -263,7 +263,8 @@ subroutine get_netcdf_var_int_1d(this, varname, var, errmsg, errcode, start, cou character(len=cl) :: file_path !Path to NetCDF file integer :: err_handling !PIO error handling code integer :: var_id !NetCDF variable ID - integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: dim_sizes(:) !Variable dimension sizes listed on file + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. integer, parameter :: var_ndims = 1 !Number of expected dimensions for variable in NetCDF file !---------------------- @@ -303,7 +304,7 @@ subroutine get_netcdf_var_int_1d(this, varname, var, errmsg, errcode, start, cou end if !Check if variable subsetting is requested and valid: - call var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, start, count) + call var_subset_check(varname, var_ndims, dim_sizes, alloc_dims, errmsg, errcode, start, count) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -312,7 +313,7 @@ subroutine get_netcdf_var_int_1d(this, varname, var, errmsg, errcode, start, cou !Now attempt to allocate and initialize variable, and !read-in the NetCDF data: - allocate(var(dim_sizes(1)), stat=errcode, errmsg=errmsg) + allocate(var(alloc_dims(1)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -2165,7 +2166,7 @@ subroutine get_dim_sizes(varname, var_rank, var_id, pio_file_handle, dim_sizes, end subroutine get_dim_sizes - subroutine var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, start, count) + subroutine var_subset_check(varname, var_ndims, dim_sizes, alloc_dims, errmsg, errcode, start, count) !Check that the start and count arrays are !either both absent or present, have the correct number !of elements, have elements that are within bounds, @@ -2178,8 +2179,9 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, star integer, intent(in) :: dim_sizes(:) !Dimension sizes for variable on file !Output arguments: - character(len=*), intent(out) :: errmsg !Error message - integer, intent(out) :: errcode !Error code + integer, allocatable, intent(out) :: alloc_dims(:) !Array that stores dimension sizes used for variable allocation + character(len=*), intent(out) :: errmsg !Error message + integer, intent(out) :: errcode !Error code !Optional input arguments: integer, optional, intent(in) :: start(:) !Start indices for each dimension @@ -2194,8 +2196,10 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, star errmsg = '' !If both start and count are not present, - !then return with no error: + !then set alloc_dims to have the same dimensionality as + !the variable on file, and return: if (.not. present(start) .and. .not. present(count)) then + allocate(alloc_dims(ndims), source=dim_sizes, stat=errcode, errmsg=errmsg) return end if @@ -2212,8 +2216,10 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, errmsg, errcode, star return end if - !Check that start has the correct number of elements: + !Determin the total number of dimensions for variable on file: file_var_dim_num = size(dim_sizes) + + !Check that start has the correct number of elements: if (size(start) /= file_var_dim_num) then errcode = bad_subset_num_elem_err write(errmsg, '(3a,i0,a,i0,a)') & From bf911952e155914d445ad1021476c683d6698f98 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 8 Aug 2025 12:07:11 -0600 Subject: [PATCH 020/150] Remove un-needed dimension index from allocate call. --- src/physics/utils/pio_reader.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 4a1ef452..dc94b932 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -2199,7 +2199,7 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, alloc_dims, errmsg, e !then set alloc_dims to have the same dimensionality as !the variable on file, and return: if (.not. present(start) .and. .not. present(count)) then - allocate(alloc_dims(ndims), source=dim_sizes, stat=errcode, errmsg=errmsg) + allocate(alloc_dims, source=dim_sizes, stat=errcode, errmsg=errmsg) return end if From bc91c132b55ff0122a70879eeeadf363f7326f8a Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 8 Aug 2025 15:09:10 -0600 Subject: [PATCH 021/150] Add bounds checking for 'count' optional variable. --- src/physics/utils/pio_reader.F90 | 26 +++++++- .../fortran/src/pio_reader/test_pio_reader.pf | 62 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index dc94b932..b741065c 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -2241,7 +2241,7 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, alloc_dims, errmsg, e !Check that start indices are within bounds: do i = 1, file_var_dim_num - if (start(i) < 0 .or. start(i) >= dim_sizes(i)) then + if ((start(i) < 0) .or. (start(i) >= dim_sizes(i))) then errcode = bad_subset_range_err write(errmsg, '(a,i0,3a,i0,a,i0,a)') & "Element ", i, " of 'start' for variable '", & @@ -2251,6 +2251,30 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, alloc_dims, errmsg, e end if end do + !Check if count has the same number of elements as the output variable: + if (size(count) == var_ndims) then + !If so, then check that count is positive and + !start + count is within bounds + !for each dimension: + do i = 1, file_var_dim_num + if (count(i) < 0) then + errcode = bad_subset_range_err + write(errmsg, '(a,i0,3a,i0,a,i0,a)') & + "Element ", i, " of 'count' for variable '", & + trim(varname), "' is out of bounds. Expected 1 to ", & + dim_sizes(i), " but got ", count(i), "." + return + else if (start(i) + count(i) > dim_sizes(i))) then + errcode = bad_subset_range_err + write(errmsg, '(a,i0,3a,i0,a,i0,a)') & + "Element ", i, " of 'start' + 'count' for variable '", & + trim(varname), "' is out of bounds. Expected 1 to ", & + dim_sizes(i), " but got ", start(i) + count(i), "." + return + end if + end do + end if + end subroutine var_subset_check end module pio_reader diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 67991697..e0b9f2a7 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1516,6 +1516,68 @@ subroutine test_pio_reader_bad_start_element_range() end subroutine test_pio_reader_bad_start_element_range +!----------------------------------------- + +@test +subroutine test_pio_reader_bad_count_element_range() + ! Check that the PIO reader raises the correct error + ! when attempting to read a variable with + ! a "count" argument that has an element out of range. + + use funit + + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t + + use ccpp_kinds, only: kind_phys + + class(abstract_netcdf_reader_t), pointer :: reader + + integer :: errcode + character(len=256) :: errmsg + + ! Variable to be read: + integer, allocatable :: pressure(:) + + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + + ! Expected error messages: + character(len=*), parameter :: expected_err_msg_neg = & + "Element 1 of 'count' for variable 'pressure' is out of bounds. Expected 1 to 59 but got -6." + + character(len=*), parameter :: expected_err_msg_huge = & + "Element 1 of 'start' + 'count' for variable 'pressure' is out of bounds. Expected 1 to 59 but got 60." + + ! Begin test: + + reader => create_netcdf_reader_t() + + ! Open file: + call reader%open_file(fname, errmsg, errcode) + + ! Attempt to read variable with negative start element: + call reader%get_var("pressure", pressure, errmsg, errcode, start=(/0/), count=(/-6/)) + + ! Check that the correct error was raised: + @assertNotEqual(0, errcode) + @assertEqual(expected_err_msg_neg, trim(errmsg)) + + ! Next attempt to read variable with too-large start element: + call reader%get_var("pressure", pressure, errmsg, errcode, start=(/1/), count=(/59/)) + + ! Check that the correct error was raised: + @assertNotEqual(0, errcode) + @assertEqual(expected_err_msg_huge, trim(errmsg)) + + ! Perform test cleanup: + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) + +end subroutine test_pio_reader_bad_count_element_range + + !++++++++++++++++++++++++++++++++++++++++++++++++++++ @test From 58a4840c56f224ed2eb842785b931c4e7ef3b044 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 8 Aug 2025 15:16:00 -0600 Subject: [PATCH 022/150] Fix syntax error. --- src/physics/utils/pio_reader.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index b741065c..0e93004b 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -2264,7 +2264,7 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, alloc_dims, errmsg, e trim(varname), "' is out of bounds. Expected 1 to ", & dim_sizes(i), " but got ", count(i), "." return - else if (start(i) + count(i) > dim_sizes(i))) then + else if (start(i) + count(i) > dim_sizes(i)) then errcode = bad_subset_range_err write(errmsg, '(a,i0,3a,i0,a,i0,a)') & "Element ", i, " of 'start' + 'count' for variable '", & From e9018c39e64764f794ed74ecb56f21fc34f8d322 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 13 Aug 2025 11:16:03 -0600 Subject: [PATCH 023/150] Add subsetting when shape(count) matches shape(var), including unit test. --- src/physics/utils/pio_reader.F90 | 29 ++++++++- .../fortran/src/pio_reader/test_pio_reader.pf | 61 +++++++++++++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 0e93004b..1544ed29 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -267,6 +267,8 @@ subroutine get_netcdf_var_int_1d(this, varname, var, errmsg, errcode, start, cou integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. integer, parameter :: var_ndims = 1 !Number of expected dimensions for variable in NetCDF file + logical, :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -304,7 +306,7 @@ subroutine get_netcdf_var_int_1d(this, varname, var, errmsg, errcode, start, cou end if !Check if variable subsetting is requested and valid: - call var_subset_check(varname, var_ndims, dim_sizes, alloc_dims, errmsg, errcode, start, count) + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -320,7 +322,15 @@ subroutine get_netcdf_var_int_1d(this, varname, var, errmsg, errcode, start, cou return end if var(:) = huge(1) - errcode = pio_get_var(pio_file_handle, var_id, var(:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_vara(pio_file_handle, var_id, start, count, var(:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -2166,7 +2176,8 @@ subroutine get_dim_sizes(varname, var_rank, var_id, pio_file_handle, dim_sizes, end subroutine get_dim_sizes - subroutine var_subset_check(varname, var_ndims, dim_sizes, alloc_dims, errmsg, errcode, start, count) + subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, & + errmsg, errcode, start, count) !Check that the start and count arrays are !either both absent or present, have the correct number !of elements, have elements that are within bounds, @@ -2179,6 +2190,7 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, alloc_dims, errmsg, e integer, intent(in) :: dim_sizes(:) !Dimension sizes for variable on file !Output arguments: + logical, intent(out) :: do_subset !True if variable subsetting will be done integer, allocatable, intent(out) :: alloc_dims(:) !Array that stores dimension sizes used for variable allocation character(len=*), intent(out) :: errmsg !Error message integer, intent(out) :: errcode !Error code @@ -2195,6 +2207,9 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, alloc_dims, errmsg, e errcode = 0 errmsg = '' + !Assume no subsetting will be done: + do_subset = .false. + !If both start and count are not present, !then set alloc_dims to have the same dimensionality as !the variable on file, and return: @@ -2273,6 +2288,14 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, alloc_dims, errmsg, e return end if end do + + !Start and Count are good, so notify caller that subsetting + !will occur, and set alloc_dims to match 'count': + do_subset = .true. + allocate(alloc_dims, source=count, stat=errcode, errmsg=errmsg) + if (errcode /= 0) then + return + end if end if end subroutine var_subset_check diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index e0b9f2a7..ef79a87e 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1251,6 +1251,67 @@ end subroutine test_pio_reader_real_get_var_bad_rank !PIO reader variable subsetting tests !+++++++++++++++++++++++++++++++++++++++++++++++++++++ +@test +subroutine test_pio_reader_1d_int_full_subset_read() + !Check that the PIO reader can read a 1D integer variable + !from a file successfully with subsetting that has + !a count array with the same rank as the output variable. + + use funit + + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t + + class(abstract_netcdf_reader_t), pointer :: reader + + integer :: errcode + character(len=256) :: errmsg + + !Variable to be read: + integer, allocatable :: pressure(:) + + + ! File name for testing: + ! NOTE: This specific file is added during the Github Actions workflow. + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + + ! Begin test: + + reader => create_netcdf_reader_t() + + ! Open file: + call reader%open_file(fname, errmsg, errcode) + + + ! Read 1D integer variable: + call reader%get_var("pressure", pressure, errmsg, errcode, start=(/0/), count=(/3/)) + + ! Check that the variable was read successfully: + @assertEqual(0, errcode) + @assertEqual('', errmsg) + + ! Check that the variable's properties are correct: + @assertTrue(allocated(pressure)) + @assertEqual(3, size(pressure)) + + ! Check that the variable's values are correct: + @assertEqual(0, pressure(1)) + @assertEqual(1, pressure(2)) + @assertEqual(2, pressure(3)) + + ! Perform test cleanup: + deallocate(pressure) + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) + +end subroutine test_pio_reader_1d_int_subset_read + +!+++++++++++++++++++++++++++++++++++++++++++++++++++++ +!PIO reader variable subsetting error tests +!+++++++++++++++++++++++++++++++++++++++++++++++++++++ + @test subroutine test_pio_reader_missing_start_argument() ! Check that the PIO reader raises the correct error From 61389fd3da107dfadce3ecf387b01ebb12c9992c Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 13 Aug 2025 11:25:01 -0600 Subject: [PATCH 024/150] Fix typo in subroutine call. --- src/physics/utils/pio_reader.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 1544ed29..56f3b031 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -326,7 +326,7 @@ subroutine get_netcdf_var_int_1d(this, varname, var, errmsg, errcode, start, cou if (do_subset) then !If subsetting is requested, then read only the specified !subset of the variable: - errcode = pio_get_vara(pio_file_handle, var_id, start, count, var(:)) + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:)) else !Otherwise, read the entire variable: errcode = pio_get_var(pio_file_handle, var_id, var(:)) From ee4a3e92a7cad5b2b05475bcb296b415e2d2ae84 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 13 Aug 2025 11:35:25 -0600 Subject: [PATCH 025/150] Fix 'do_subset' syntax error. --- src/physics/utils/pio_reader.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 56f3b031..e18b1dfa 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -267,8 +267,8 @@ subroutine get_netcdf_var_int_1d(this, varname, var, errmsg, errcode, start, cou integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. integer, parameter :: var_ndims = 1 !Number of expected dimensions for variable in NetCDF file - logical, :: do_subset !Will variable subsetting be done? Answer provided by - !var_subset_check subroutine. + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: From 249f87acc79f0df035490b3e4be0044e8fd917b2 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 13 Aug 2025 11:44:41 -0600 Subject: [PATCH 026/150] Fix bad subroutine name. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index ef79a87e..68f24d39 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1306,7 +1306,7 @@ subroutine test_pio_reader_1d_int_full_subset_read() deallocate(reader) nullify(reader) -end subroutine test_pio_reader_1d_int_subset_read +end subroutine test_pio_reader_1d_int_full_subset_read !+++++++++++++++++++++++++++++++++++++++++++++++++++++ !PIO reader variable subsetting error tests From c47dbe6579976f844bd50b84cf1e94b0504e8ed8 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 13 Aug 2025 11:52:30 -0600 Subject: [PATCH 027/150] Flip error message and error code check for subset read test. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 68f24d39..b827cecb 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1288,8 +1288,8 @@ subroutine test_pio_reader_1d_int_full_subset_read() call reader%get_var("pressure", pressure, errmsg, errcode, start=(/0/), count=(/3/)) ! Check that the variable was read successfully: - @assertEqual(0, errcode) @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Check that the variable's properties are correct: @assertTrue(allocated(pressure)) From 5175ea33340905cde0f33337c088e9ed48e26396 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 13 Aug 2025 12:56:15 -0600 Subject: [PATCH 028/150] Add debugging output. --- src/physics/utils/pio_reader.F90 | 2 ++ test/unit/fortran/src/pio_reader/test_pio_reader.pf | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index e18b1dfa..3e3356f8 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -326,6 +326,8 @@ subroutine get_netcdf_var_int_1d(this, varname, var, errmsg, errcode, start, cou if (do_subset) then !If subsetting is requested, then read only the specified !subset of the variable: + write(*,*) 'var size:', size(var), alloc_dims(:) + write(*,*) 'start/count:', start(:), count(:) errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:)) else !Otherwise, read the entire variable: diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index b827cecb..965b3c0e 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1285,7 +1285,7 @@ subroutine test_pio_reader_1d_int_full_subset_read() ! Read 1D integer variable: - call reader%get_var("pressure", pressure, errmsg, errcode, start=(/0/), count=(/3/)) + call reader%get_var("pressure", pressure, errmsg, errcode, start=(/1/), count=(/3/)) ! Check that the variable was read successfully: @assertEqual('', errmsg) From a45e9d2536617032d13a8abf3cf7bae57e41aeb9 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 13 Aug 2025 14:17:05 -0600 Subject: [PATCH 029/150] Cleanup debug output and modify start/count element range checks. --- src/physics/utils/pio_reader.F90 | 24 +++++++++---------- .../fortran/src/pio_reader/test_pio_reader.pf | 4 ++-- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 3e3356f8..d6f261b0 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -326,8 +326,6 @@ subroutine get_netcdf_var_int_1d(this, varname, var, errmsg, errcode, start, cou if (do_subset) then !If subsetting is requested, then read only the specified !subset of the variable: - write(*,*) 'var size:', size(var), alloc_dims(:) - write(*,*) 'start/count:', start(:), count(:) errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:)) else !Otherwise, read the entire variable: @@ -2258,12 +2256,12 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims !Check that start indices are within bounds: do i = 1, file_var_dim_num - if ((start(i) < 0) .or. (start(i) >= dim_sizes(i))) then + if ((start(i) < 1) .or. (start(i) > dim_sizes(i))) then errcode = bad_subset_range_err write(errmsg, '(a,i0,3a,i0,a,i0,a)') & "Element ", i, " of 'start' for variable '", & - trim(varname), "' is out of bounds. Expected 0 to ", & - dim_sizes(i)-1, " but got ", start(i), "." + trim(varname), "' is out of bounds. Expected 1 to ", & + dim_sizes(i), " but got ", start(i), "." return end if end do @@ -2274,20 +2272,20 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims !start + count is within bounds !for each dimension: do i = 1, file_var_dim_num - if (count(i) < 0) then + if ((count(i) < 1) .or. (count(i) > dim_sizes(i))) then errcode = bad_subset_range_err write(errmsg, '(a,i0,3a,i0,a,i0,a)') & "Element ", i, " of 'count' for variable '", & trim(varname), "' is out of bounds. Expected 1 to ", & dim_sizes(i), " but got ", count(i), "." return - else if (start(i) + count(i) > dim_sizes(i)) then - errcode = bad_subset_range_err - write(errmsg, '(a,i0,3a,i0,a,i0,a)') & - "Element ", i, " of 'start' + 'count' for variable '", & - trim(varname), "' is out of bounds. Expected 1 to ", & - dim_sizes(i), " but got ", start(i) + count(i), "." - return +! else if (start(i) + count(i) > dim_sizes(i)) then +! errcode = bad_subset_range_err +! write(errmsg, '(a,i0,3a,i0,a,i0,a)') & +! "Element ", i, " of 'start' + 'count' for variable '", & +! trim(varname), "' is out of bounds. Expected 1 to ", & +! dim_sizes(i), " but got ", start(i) + count(i), "." +! return end if end do diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 965b3c0e..0d7887eb 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1618,14 +1618,14 @@ subroutine test_pio_reader_bad_count_element_range() call reader%open_file(fname, errmsg, errcode) ! Attempt to read variable with negative start element: - call reader%get_var("pressure", pressure, errmsg, errcode, start=(/0/), count=(/-6/)) + call reader%get_var("pressure", pressure, errmsg, errcode, start=(/1/), count=(/-6/)) ! Check that the correct error was raised: @assertNotEqual(0, errcode) @assertEqual(expected_err_msg_neg, trim(errmsg)) ! Next attempt to read variable with too-large start element: - call reader%get_var("pressure", pressure, errmsg, errcode, start=(/1/), count=(/59/)) + call reader%get_var("pressure", pressure, errmsg, errcode, start=(/1/), count=(/60/)) ! Check that the correct error was raised: @assertNotEqual(0, errcode) From 53780e7184035f7dc9336cf7d79bab15178ae3cf Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 13 Aug 2025 16:02:56 -0600 Subject: [PATCH 030/150] Add subsetting to 2-D integer interface, add unit test, fix bad unit test error message, flip errmsg/errcode, and add variable declarations. --- src/physics/utils/pio_reader.F90 | 80 ++++++++++++++++- .../fortran/src/pio_reader/test_pio_reader.pf | 86 ++++++++++++++++--- 2 files changed, 151 insertions(+), 15 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index d6f261b0..5ad00253 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -160,6 +160,11 @@ subroutine get_netcdf_var_int_0d(this, varname, var, errmsg, errcode, start, cou integer :: var_id !NetCDF variable ID integer :: ndims !Number of variable dimensions on NetCDF file integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. + + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. + !---------------------- !Check if file is open: @@ -375,8 +380,11 @@ subroutine get_netcdf_var_int_2d(this, varname, var, errmsg, errcode, start, cou integer :: err_handling !PIO error handling code integer :: var_id !NetCDF variable ID integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. integer, parameter :: var_ndims = 2 !Number of expected dimensions for variable in NetCDF file + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -413,16 +421,32 @@ subroutine get_netcdf_var_int_2d(this, varname, var, errmsg, errcode, start, cou return end if + !Check if variable subsetting is requested and valid: + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and !read-in the NetCDF data: - allocate(var(dim_sizes(1), dim_sizes(2)), stat=errcode, errmsg=errmsg) + allocate(var(alloc_dims(1), alloc_dims(2)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) return end if var(:,:) = huge(1) - errcode = pio_get_var(pio_file_handle, var_id, var(:,:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:,:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:,:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -467,8 +491,11 @@ subroutine get_netcdf_var_int_3d(this, varname, var, errmsg, errcode, start, cou integer :: err_handling !PIO error handling code integer :: var_id !NetCDF variable ID integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. integer, parameter :: var_ndims = 3 !Number of expected dimensions for variable in NetCDF file + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -561,8 +588,11 @@ subroutine get_netcdf_var_int_4d(this, varname, var, errmsg, errcode, start, cou integer :: ndims !Number of variable dimensions on NetCDF file integer, allocatable :: dim_ids(:) !Variable dimension IDs integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. integer, parameter :: var_ndims = 4 !Number of expected dimensions for variable in NetCDF file + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -653,8 +683,11 @@ subroutine get_netcdf_var_int_5d(this, varname, var, errmsg, errcode, start, cou integer :: err_handling !PIO error handling code integer :: var_id !NetCDF variable ID integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. integer, parameter :: var_ndims = 5 !Number of expected dimensions for variable in NetCDF file + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -751,6 +784,10 @@ subroutine get_netcdf_var_real_0d(this, varname, var, errmsg, errcode, start, co integer :: var_id !NetCDF variable ID integer :: ndims !Number of variable dimensions on NetCDF file integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. + + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -855,8 +892,11 @@ subroutine get_netcdf_var_real_1d(this, varname, var, errmsg, errcode, start, co integer :: err_handling !PIO error handling code integer :: var_id !NetCDF variable ID integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. integer, parameter :: var_ndims = 1 !Number of expected dimensions for variable in NetCDF file + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -947,8 +987,11 @@ subroutine get_netcdf_var_real_2d(this, varname, var, errmsg, errcode, start, co integer :: err_handling !PIO error handling code integer :: var_id !NetCDF variable ID integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. integer, parameter :: var_ndims = 2 !Number of expected dimensions for variable in NetCDF file + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -1039,8 +1082,11 @@ subroutine get_netcdf_var_real_3d(this, varname, var, errmsg, errcode, start, co integer :: err_handling !PIO error handling code integer :: var_id !NetCDF variable ID integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. integer, parameter :: var_ndims = 3 !Number of expected dimensions for variable in NetCDF file + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -1131,8 +1177,11 @@ subroutine get_netcdf_var_real_4d(this, varname, var, errmsg, errcode, start, co integer :: err_handling !PIO error handling code integer :: var_id !NetCDF variable ID integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. integer, parameter :: var_ndims = 4 !Number of expected dimensions for variable in NetCDF file + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -1223,8 +1272,11 @@ subroutine get_netcdf_var_real_5d(this, varname, var, errmsg, errcode, start, co integer :: err_handling !PIO error handling code integer :: var_id !NetCDF variable ID integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. integer, parameter :: var_ndims = 5 !Number of expected dimensions for variable in NetCDF file + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -1322,6 +1374,7 @@ subroutine get_netcdf_var_char_0d(this, varname, var, errmsg, errcode, start, co integer :: var_id !NetCDF variable ID integer :: nc_type !NetCDF variable type integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. !NOTE: NetCDF supports both character arrays and string-type !data depending on the NetCDF version, so the dimensions @@ -1331,6 +1384,9 @@ subroutine get_netcdf_var_char_0d(this, varname, var, errmsg, errcode, start, co !differently, but for now just confirm it's a character !array and check for ndims = rank+1 integer, parameter :: var_ndims = 1 !Number of expected dimensions for variable in NetCDF file + + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -1446,6 +1502,7 @@ subroutine get_netcdf_var_char_1d(this, varname, var, errmsg, errcode, start, co integer :: var_id !NetCDF variable ID integer :: nc_type !NetCDF variable type integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. !NOTE: NetCDF supports both character arrays and string-type !data depending on the NetCDF version, so the dimensions @@ -1455,6 +1512,9 @@ subroutine get_netcdf_var_char_1d(this, varname, var, errmsg, errcode, start, co !differently, but for now just confirm it's a character !array and check for ndims = rank+1 integer, parameter :: var_ndims = 2 !Number of expected dimensions for variable in NetCDF file + + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -1570,6 +1630,7 @@ subroutine get_netcdf_var_char_2d(this, varname, var, errmsg, errcode, start, co integer :: var_id !NetCDF variable ID integer :: nc_type !NetCDF variable type integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. !NOTE: NetCDF supports both character arrays and string-type !data depending on the NetCDF version, so the dimensions @@ -1579,6 +1640,9 @@ subroutine get_netcdf_var_char_2d(this, varname, var, errmsg, errcode, start, co !differently, but for now just confirm it's a character !array and check for ndims = rank+1 integer, parameter :: var_ndims = 3 !Number of expected dimensions for variable in NetCDF file + + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -1694,6 +1758,7 @@ subroutine get_netcdf_var_char_3d(this, varname, var, errmsg, errcode, start, co integer :: var_id !NetCDF variable ID integer :: nc_type !NetCDF variable type integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. !NOTE: NetCDF supports both character arrays and string-type !data depending on the NetCDF version, so the dimensions @@ -1703,6 +1768,9 @@ subroutine get_netcdf_var_char_3d(this, varname, var, errmsg, errcode, start, co !differently, but for now just confirm it's a character !array and check for ndims = rank+1 integer, parameter :: var_ndims = 4 !Number of expected dimensions for variable in NetCDF file + + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- @@ -1820,6 +1888,7 @@ subroutine get_netcdf_var_char_4d(this, varname, var, errmsg, errcode, start, co integer :: var_id !NetCDF variable ID integer :: nc_type !NetCDF variable type integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. !NOTE: NetCDF supports both character arrays and string-type !data depending on the NetCDF version, so the dimensions @@ -1829,6 +1898,9 @@ subroutine get_netcdf_var_char_4d(this, varname, var, errmsg, errcode, start, co !differently, but for now just confirm it's a character !array and check for ndims = rank+1 integer, parameter :: var_ndims = 5 !Number of expected dimensions for variable in NetCDF file + + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: @@ -1945,6 +2017,7 @@ subroutine get_netcdf_var_char_5d(this, varname, var, errmsg, errcode, start, co integer :: var_id !NetCDF variable ID integer :: nc_type !NetCDF variable type integer, allocatable :: dim_sizes(:) !Variable dimension sizes + integer, allocatable :: alloc_dims(:) !Variable dimension sizes to allocate output variable to. !NOTE: NetCDF supports both character arrays and string-type !data depending on the NetCDF version, so the dimensions @@ -1954,6 +2027,9 @@ subroutine get_netcdf_var_char_5d(this, varname, var, errmsg, errcode, start, co !differently, but for now just confirm it's a character !array and check for ndims = rank+1 integer, parameter :: var_ndims = 6 !Number of expected dimensions for variable in NetCDF file + + logical :: do_subset !Will variable subsetting be done? Answer provided by + !var_subset_check subroutine. !---------------------- !Check if file is open: diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 0d7887eb..7a0b8353 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -53,22 +53,22 @@ subroutine test_pio_reader_open_close_file() call reader%open_file(fname, errmsg, errcode) ! Check that file was opened successfully: - @assertEqual(0, errcode) @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Attempt to close file: call reader%close_file(errmsg, errcode) ! Check that file was closed successfully: - @assertEqual(0, errcode) @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Attempt to close the file again: call reader%close_file(errmsg, errcode) ! Check that closing an already-closed file does not cause an error: - @assertEqual(0, errcode) @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Deallocate reader object: deallocate(reader) @@ -208,8 +208,8 @@ subroutine test_pio_reader_1d_int_read() call reader%get_var("pressure", pressure, errmsg, errcode) ! Check that the variable was read successfully: - @assertEqual(0, errcode) @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Check that the variable's properties are correct: @assertTrue(allocated(pressure)) @@ -262,8 +262,8 @@ subroutine test_pio_reader_2d_int_read() call reader%get_var("minor_limits_gpt_lower", minor_limits_gpt_lower, errmsg, errcode) ! Check that the variable was read successfully: - @assertEqual(0, errcode) @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Check that the variable's properties are correct: @assertTrue(allocated(minor_limits_gpt_lower)) @@ -317,8 +317,8 @@ subroutine test_pio_reader_3d_int_read() call reader%get_var("key_species", key_species, errmsg, errcode) ! Check that the variable was read successfully: - @assertEqual(0, errcode) @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Check that the variable's properties are correct: @assertTrue(allocated(key_species)) @@ -375,8 +375,8 @@ subroutine test_pio_reader_1d_char_read() call reader%get_var("gas_names", gas_names, errmsg, errcode) ! Check that the variable was read successfully: - @assertEqual(0, errcode) @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Check that the variable's properties are correct: @assertTrue(allocated(gas_names)) @@ -453,8 +453,8 @@ subroutine test_pio_reader_0d_real_read() call reader%get_var("press_ref_trop", press_ref_trop, errmsg, errcode) ! Check that the variable was read successfully: - @assertEqual(0, errcode) @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Check that the variable's properties are correct: @assertTrue(allocated(press_ref_trop)) @@ -511,8 +511,8 @@ subroutine test_pio_reader_1d_real_read() call reader%get_var("press_ref", press_ref, errmsg, errcode) ! Check that the variable was read successfully: - @assertEqual(0, errcode) @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Check that the variable's properties are correct: @assertTrue(allocated(press_ref)) @@ -571,8 +571,8 @@ subroutine test_pio_reader_2d_real_read() call reader%get_var("bnd_limits_wavenumber", bnd_limits_wavenumber, errmsg, errcode) ! Check that the variable was read successfully: - @assertEqual(0, errcode) @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Check that the variable's properties are correct: @assertTrue(allocated(bnd_limits_wavenumber)) @@ -634,8 +634,8 @@ subroutine test_pio_reader_3d_real_read() call reader%get_var("vmr_ref", vmr_ref, errmsg, errcode) ! Check that the variable was read successfully: - @assertEqual(0, errcode) @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Check that the variable's properties are correct: @assertTrue(allocated(vmr_ref)) @@ -715,8 +715,8 @@ subroutine test_pio_reader_4d_real_read_with_dims() call reader%get_var("kmajor", kmajor, errmsg, errcode) ! Check that the variable was read successfully: - @assertEqual(0, errcode) @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Check that the variable's properties are correct: @@ -1308,6 +1308,66 @@ subroutine test_pio_reader_1d_int_full_subset_read() end subroutine test_pio_reader_1d_int_full_subset_read +!---------------------------------------- + +@test +subroutine test_pio_reader_2d_int_full_subset_read() + !Check that the PIO reader can read a 2D integer variable + !from a file successfully. + + use funit + + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t + + class(abstract_netcdf_reader_t), pointer :: reader + + integer :: errcode + character(len=256) :: errmsg + + !Variable to be read: + integer, allocatable :: minor_limits_gpt_lower(:,:) + + ! File name for testing: + ! NOTE: This specific file is added during the Github Actions workflow. + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + + ! Begin test: + + reader => create_netcdf_reader_t() + + ! Open file: + call reader%open_file(fname, errmsg, errcode) + + ! Read 2D integer variable: + call reader%get_var("minor_limits_gpt_lower", minor_limits_gpt_lower, errmsg, errcode, & + start=(/1,2/), count=(/2,3/)) + + ! Check that the variable was read successfully: + @assertEqual('', errmsg) + @assertEqual(0, errcode) + + ! Check that the variable's properties are correct: + @assertTrue(allocated(minor_limits_gpt_lower)) + @assertEqual(2, size(minor_limits_gpt_lower, dim=1)) + @assertEqual(3, size(minor_limits_gpt_lower, dim=2)) + + ! Check that the variable's values are correct: + !@assertEqual(1, sum(minor_limits_gpt_lower(1,1))) + !@assertEqual(10, sum(minor_limits_gpt_lower(2,1))) + !@assertEqual(2, sum(minor_limits_gpt_lower(1,2))) + + write(*,*) 'minor_limits_gpt_lower values:', minor_limits_gpt_lower(:, :) + + ! Perform test cleanup: + deallocate(minor_limits_gpt_lower) + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) + +end subroutine test_pio_reader_2d_int_full_subset_read + !+++++++++++++++++++++++++++++++++++++++++++++++++++++ !PIO reader variable subsetting error tests !+++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -1608,7 +1668,7 @@ subroutine test_pio_reader_bad_count_element_range() "Element 1 of 'count' for variable 'pressure' is out of bounds. Expected 1 to 59 but got -6." character(len=*), parameter :: expected_err_msg_huge = & - "Element 1 of 'start' + 'count' for variable 'pressure' is out of bounds. Expected 1 to 59 but got 60." + "Element 1 of 'count' for variable 'pressure' is out of bounds. Expected 1 to 59 but got 60." ! Begin test: From 41fcd3d8c1f683449b2b581affab7ddf9c448a2f Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 13 Aug 2025 16:13:09 -0600 Subject: [PATCH 031/150] Add 'singe value' subset test, and fix bad error message. --- .../fortran/src/pio_reader/test_pio_reader.pf | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 7a0b8353..901ec12e 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1354,11 +1354,26 @@ subroutine test_pio_reader_2d_int_full_subset_read() @assertEqual(3, size(minor_limits_gpt_lower, dim=2)) ! Check that the variable's values are correct: - !@assertEqual(1, sum(minor_limits_gpt_lower(1,1))) - !@assertEqual(10, sum(minor_limits_gpt_lower(2,1))) - !@assertEqual(2, sum(minor_limits_gpt_lower(1,2))) + @assertEqual(1, sum(minor_limits_gpt_lower(1,1))) + @assertEqual(10, sum(minor_limits_gpt_lower(2,1))) + @assertEqual(1, sum(minor_limits_gpt_lower(1,2))) + @assertEqual(10, sum(minor_limits_gpt_lower(2,2))) + @assertEqual(1, sum(minor_limits_gpt_lower(1,3))) + @assertEqual(10, sum(minor_limits_gpt_lower(2,3))) + + ! Now deallocate the variable and try a single value subset: + deallocate(minor_limits_gpt_lower) + + call reader%get_var("minor_limits_gpt_lower", minor_limits_gpt_lower, errmsg, errcode, & + start=(/1,6/), count=(/1,1/)) - write(*,*) 'minor_limits_gpt_lower values:', minor_limits_gpt_lower(:, :) + ! Check that the variable's properties are correct: + @assertTrue(allocated(minor_limits_gpt_lower)) + @assertEqual(1, size(minor_limits_gpt_lower, dim=1)) + @assertEqual(1, size(minor_limits_gpt_lower, dim=2)) + + ! Check that the variable's values are correct: + @assertEqual(11, minor_limits_gpt_lower(1,1)) ! Perform test cleanup: deallocate(minor_limits_gpt_lower) @@ -1604,10 +1619,10 @@ subroutine test_pio_reader_bad_start_element_range() ! Expected error messages: character(len=*), parameter :: expected_err_msg_neg = & - "Element 1 of 'start' for variable 'pressure' is out of bounds. Expected 0 to 58 but got -1." + "Element 1 of 'start' for variable 'pressure' is out of bounds. Expected 1 to 59 but got -1." character(len=*), parameter :: expected_err_msg_huge = & - "Element 1 of 'start' for variable 'pressure' is out of bounds. Expected 0 to 58 but got 200." + "Element 1 of 'start' for variable 'pressure' is out of bounds. Expected 1 to 59 but got 200." ! Begin test: From 640a440abeb20938d92253d5f93d8936263da6c2 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 13 Aug 2025 16:18:18 -0600 Subject: [PATCH 032/150] remove extraneous sum calls. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 901ec12e..a4a05426 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1354,12 +1354,12 @@ subroutine test_pio_reader_2d_int_full_subset_read() @assertEqual(3, size(minor_limits_gpt_lower, dim=2)) ! Check that the variable's values are correct: - @assertEqual(1, sum(minor_limits_gpt_lower(1,1))) - @assertEqual(10, sum(minor_limits_gpt_lower(2,1))) - @assertEqual(1, sum(minor_limits_gpt_lower(1,2))) - @assertEqual(10, sum(minor_limits_gpt_lower(2,2))) - @assertEqual(1, sum(minor_limits_gpt_lower(1,3))) - @assertEqual(10, sum(minor_limits_gpt_lower(2,3))) + @assertEqual(1, minor_limits_gpt_lower(1,1)) + @assertEqual(10, minor_limits_gpt_lower(2,1)) + @assertEqual(1, minor_limits_gpt_lower(1,2)) + @assertEqual(10, minor_limits_gpt_lower(2,2)) + @assertEqual(1, minor_limits_gpt_lower(1,3)) + @assertEqual(10, minor_limits_gpt_lower(2,3)) ! Now deallocate the variable and try a single value subset: deallocate(minor_limits_gpt_lower) From 5c480756f6d7d7990517c559319d088c8266e3d5 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 14 Aug 2025 10:55:39 -0600 Subject: [PATCH 033/150] Move non-subsetting rank check to 'var_subset_check', and add additional 'count' size checks. --- src/physics/utils/pio_reader.F90 | 106 +++++++++++++++++++------------ 1 file changed, 64 insertions(+), 42 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 5ad00253..cacb3b78 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -303,7 +303,7 @@ subroutine get_netcdf_var_int_1d(this, varname, var, errmsg, errcode, start, cou end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -414,7 +414,7 @@ subroutine get_netcdf_var_int_2d(this, varname, var, errmsg, errcode, start, cou end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -525,7 +525,7 @@ subroutine get_netcdf_var_int_3d(this, varname, var, errmsg, errcode, start, cou end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -622,7 +622,7 @@ subroutine get_netcdf_var_int_4d(this, varname, var, errmsg, errcode, start, cou end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -717,7 +717,7 @@ subroutine get_netcdf_var_int_5d(this, varname, var, errmsg, errcode, start, cou end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -926,7 +926,7 @@ subroutine get_netcdf_var_real_1d(this, varname, var, errmsg, errcode, start, co end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1021,7 +1021,7 @@ subroutine get_netcdf_var_real_2d(this, varname, var, errmsg, errcode, start, co end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1116,7 +1116,7 @@ subroutine get_netcdf_var_real_3d(this, varname, var, errmsg, errcode, start, co end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1211,7 +1211,7 @@ subroutine get_netcdf_var_real_4d(this, varname, var, errmsg, errcode, start, co end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1306,7 +1306,7 @@ subroutine get_netcdf_var_real_5d(this, varname, var, errmsg, errcode, start, co end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1437,7 +1437,7 @@ subroutine get_netcdf_var_char_0d(this, varname, var, errmsg, errcode, start, co end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1565,7 +1565,7 @@ subroutine get_netcdf_var_char_1d(this, varname, var, errmsg, errcode, start, co end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1693,7 +1693,7 @@ subroutine get_netcdf_var_char_2d(this, varname, var, errmsg, errcode, start, co end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1823,7 +1823,7 @@ subroutine get_netcdf_var_char_3d(this, varname, var, errmsg, errcode, start, co end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1951,7 +1951,7 @@ subroutine get_netcdf_var_char_4d(this, varname, var, errmsg, errcode, start, co end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -2080,7 +2080,7 @@ subroutine get_netcdf_var_char_5d(this, varname, var, errmsg, errcode, start, co end if !Get variable dimension sizes: - call get_dim_sizes(varname, var_ndims, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + call get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -2171,7 +2171,7 @@ subroutine get_pio_errmsg(caller_errcode, varname, errcode, errmsg, file_msg) end if end subroutine get_pio_errmsg - subroutine get_dim_sizes(varname, var_rank, var_id, pio_file_handle, dim_sizes, errcode, errmsg) + subroutine get_dim_sizes(varname, var_id, pio_file_handle, dim_sizes, errcode, errmsg) !Extract the dimension sizes for a NetCDF !variable given a variable ID, and !return a new 1-D array with the variable @@ -2183,7 +2183,6 @@ subroutine get_dim_sizes(varname, var_rank, var_id, pio_file_handle, dim_sizes, !Input variables: character(len=*), intent(in) :: varname !Name of NetCDF variable - integer, intent(in) :: var_rank !Rank (number of dims) of NetCDF variable integer, intent(in) :: var_id !Variable ID for NetCDF variable type(file_desc_t), intent(in) :: pio_file_handle !File handle type used by PIO @@ -2210,15 +2209,6 @@ subroutine get_dim_sizes(varname, var_rank, var_id, pio_file_handle, dim_sizes, return end if - !Check that the variable rank as specified by the caller - !matches what is found on the NetCDF file: - errcode = 0 - if(ndims /= var_rank) then - errcode = bad_var_rank_err - errmsg = "Variable '"//trim(varname)//"' isn't declared with the correct number of dimensions" - return - end if - !Get variable dimension sizes: !Allocate NetCDF variable dimension ID array: allocate(dim_ids(ndims), stat=errcode, errmsg=errmsg) @@ -2276,8 +2266,9 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims integer, optional, intent(in) :: count(:) !Number of elements to read for each dimension !Local variables: - integer :: file_var_dim_num !Number of dimensions for variable on file - integer :: i !Loop control variable + integer :: file_var_dim_num !Number of dimensions for variable on file + integer :: count_true_dim_num !Number of dimensions in count array that are greater than 1 + integer :: i !Loop control variable !Initialize error code and message: errcode = 0 @@ -2286,12 +2277,28 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims !Assume no subsetting will be done: do_subset = .false. + !Determin the total number of dimensions for variable on file: + file_var_dim_num = size(dim_sizes) + !If both start and count are not present, - !then set alloc_dims to have the same dimensionality as - !the variable on file, and return: + !then check for matching ranks, and set alloc_dims + !to have the same dimensionality as the file variable: if (.not. present(start) .and. .not. present(count)) then + + !Check that the variable rank as specified by the caller + !matches what is found on the NetCDF file: + if(file_var_dim_num /= var_ndims) then + errcode = bad_var_rank_err + write(errmsg, '(4a,i0,a,i0,a)') & + "Variable '",trim(varname),"' isn't declared with the correct number of dimensions.", & + " Expected ", file_var_dim_num, " dimensions, but is declared with ", & + var_ndims, " dimensions." + return + end if + + !Allocate "alloc_dims" to exactly match the file variable dimensions: allocate(alloc_dims, source=dim_sizes, stat=errcode, errmsg=errmsg) - return + return !Nothing more to do here. end if !At this point, either start or count @@ -2307,9 +2314,6 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims return end if - !Determin the total number of dimensions for variable on file: - file_var_dim_num = size(dim_sizes) - !Check that start has the correct number of elements: if (size(start) /= file_var_dim_num) then errcode = bad_subset_num_elem_err @@ -2355,13 +2359,6 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims trim(varname), "' is out of bounds. Expected 1 to ", & dim_sizes(i), " but got ", count(i), "." return -! else if (start(i) + count(i) > dim_sizes(i)) then -! errcode = bad_subset_range_err -! write(errmsg, '(a,i0,3a,i0,a,i0,a)') & -! "Element ", i, " of 'start' + 'count' for variable '", & -! trim(varname), "' is out of bounds. Expected 1 to ", & -! dim_sizes(i), " but got ", start(i) + count(i), "." -! return end if end do @@ -2372,6 +2369,31 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims if (errcode /= 0) then return end if + else if (size(count) > var_ndims) then + !The subsetting appears to be reducing + !the dimensionality of the file variable. + !Thus make sure that the number of dimensions + !with more than a count of "1" matches + !the output variable's dimensionality: + count_true_dim_num = count(count(:) > 1) !Notethat there is both a "count" function and "count" array + + if (count_true_dim_num > var_ndims) then + errcode = bad_subset_num_elem_err + write(errmsg, '(3a,i0,a,i0,a)') & + "The 'count' array for variable '", trim(varname), & + "' has too many elements with a count greater than one. Expected at most ", & + var_ndims, " but got ", count_true_dim_num, "." + return + end if + + else if (size(count) < var_ndims) then + !I don't think this should ever happen as + !we have already checked that count matches + !file_var_dim num, but just in case let's + !throw an error here: + errcode = bad_subset_num_elem_err + errmsg = 'The number of elements in the 'count' array for variable "'//trim(varname)//'" has too few dimensions.' + return end if end subroutine var_subset_check From d6fc64942bf2a1b4228ba79c0476b9caf8e19547 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 14 Aug 2025 11:09:53 -0600 Subject: [PATCH 034/150] Remove use of 'count' intrinsic to avoid name collision. --- src/physics/utils/pio_reader.F90 | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index cacb3b78..281cbc76 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -2366,24 +2366,28 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims !will occur, and set alloc_dims to match 'count': do_subset = .true. allocate(alloc_dims, source=count, stat=errcode, errmsg=errmsg) - if (errcode /= 0) then - return - end if + return !Nothing more to do here. + else if (size(count) > var_ndims) then !The subsetting appears to be reducing !the dimensionality of the file variable. !Thus make sure that the number of dimensions !with more than a count of "1" matches !the output variable's dimensionality: - count_true_dim_num = count(count(:) > 1) !Notethat there is both a "count" function and "count" array + count_true_dim_num = 0 + do i=1, size(count) + if (count(i) > 1) then + count_true_dim_num = count_true_dim_num + 1 + end if + end do if (count_true_dim_num > var_ndims) then - errcode = bad_subset_num_elem_err - write(errmsg, '(3a,i0,a,i0,a)') & - "The 'count' array for variable '", trim(varname), & - "' has too many elements with a count greater than one. Expected at most ", & - var_ndims, " but got ", count_true_dim_num, "." - return + errcode = bad_subset_num_elem_err + write(errmsg, '(3a,i0,a,i0,a)') & + "The 'count' array for variable '", trim(varname), & + "' has too many elements with a count greater than one. Expected at most ", & + var_ndims, " but got ", count_true_dim_num, "." + return end if else if (size(count) < var_ndims) then From de647be7b200ee5a62863bd6502cd68430c1a30c Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 14 Aug 2025 11:30:36 -0600 Subject: [PATCH 035/150] Fix bad syntax in error message. --- src/physics/utils/pio_reader.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 281cbc76..cbf8d3dd 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -2396,7 +2396,7 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims !file_var_dim num, but just in case let's !throw an error here: errcode = bad_subset_num_elem_err - errmsg = 'The number of elements in the 'count' array for variable "'//trim(varname)//'" has too few dimensions.' + errmsg = "The number of elements in the 'count' array for variable '"//trim(varname)//"' has too few dimensions." return end if From 3385d54b195d2ce4235392f580a605afcb30c877 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 14 Aug 2025 14:19:54 -0600 Subject: [PATCH 036/150] Add subsetting calls to all non-scalar get_var calls. --- src/physics/utils/pio_reader.F90 | 168 +++++++++++++++--- .../fortran/src/pio_reader/test_pio_reader.pf | 8 +- 2 files changed, 146 insertions(+), 30 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index cbf8d3dd..8a136fd5 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -532,9 +532,17 @@ subroutine get_netcdf_var_int_3d(this, varname, var, errmsg, errcode, start, cou return end if + !Check if variable subsetting is requested and valid: + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and !read-in the NetCDF data: - allocate(var(dim_sizes(1), dim_sizes(2), dim_sizes(3)), stat=errcode, errmsg=errmsg) + allocate(var(alloc_dims(1), alloc_dims(2), alloc_dims(3)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -629,9 +637,17 @@ subroutine get_netcdf_var_int_4d(this, varname, var, errmsg, errcode, start, cou return end if + !Check if variable subsetting is requested and valid: + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and !read-in the NetCDF data: - allocate(var(dim_sizes(1), dim_sizes(2), dim_sizes(3), dim_sizes(4)), stat=errcode, errmsg=errmsg) + allocate(var(alloc_dims(1), alloc_dims(2), alloc_dims(3), alloc_dims(4)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -724,9 +740,17 @@ subroutine get_netcdf_var_int_5d(this, varname, var, errmsg, errcode, start, cou return end if + !Check if variable subsetting is requested and valid: + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and !read-in the NetCDF data: - allocate(var(dim_sizes(1), dim_sizes(2), dim_sizes(3), dim_sizes(4), dim_sizes(5)), & + allocate(var(dalloc_dims(1), alloc_dims(2), alloc_dims(3), alloc_dims(4), alloc_dims(5)), & stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: @@ -933,9 +957,17 @@ subroutine get_netcdf_var_real_1d(this, varname, var, errmsg, errcode, start, co return end if + !Check if variable subsetting is requested and valid: + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and !read-in the NetCDF data: - allocate(var(dim_sizes(1)), stat=errcode, errmsg=errmsg) + allocate(var(alloc_dims(1)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1028,9 +1060,17 @@ subroutine get_netcdf_var_real_2d(this, varname, var, errmsg, errcode, start, co return end if + !Check if variable subsetting is requested and valid: + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and !read-in the NetCDF data: - allocate(var(dim_sizes(1), dim_sizes(2)), stat=errcode, errmsg=errmsg) + allocate(var(alloc_dims(1), alloc_dims(2)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1123,9 +1163,17 @@ subroutine get_netcdf_var_real_3d(this, varname, var, errmsg, errcode, start, co return end if + !Check if variable subsetting is requested and valid: + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize the variable, !and read-in the NetCDF data: - allocate(var(dim_sizes(1), dim_sizes(2), dim_sizes(3)), stat=errcode, errmsg=errmsg) + allocate(var(alloc_dims(1), alloc_dims(2), alloc_dims(3)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1218,9 +1266,17 @@ subroutine get_netcdf_var_real_4d(this, varname, var, errmsg, errcode, start, co return end if + !Check if variable subsetting is requested and valid: + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and !read-in the NetCDF data: - allocate(var(dim_sizes(1), dim_sizes(2), dim_sizes(3), dim_sizes(4)), stat=errcode, errmsg=errmsg) + allocate(var(alloc_dims(1), alloc_dims(2), alloc_dims(3), alloc_dims(4)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1313,6 +1369,14 @@ subroutine get_netcdf_var_real_5d(this, varname, var, errmsg, errcode, start, co return end if + !Check if variable subsetting is requested and valid: + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize the variable, !and read-in the NetCDF data: allocate(var(dim_sizes(1), dim_sizes(2), dim_sizes(3), dim_sizes(4), dim_sizes(5)), & @@ -1572,11 +1636,21 @@ subroutine get_netcdf_var_char_1d(this, varname, var, errmsg, errcode, start, co return end if + !Check if variable subsetting is requested and valid + !Note that this involves ignoring the first dimension, + !which is just the character length: + call var_subset_check(varname, var_ndims, dim_sizes(2:), do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and - !read-in the NetCDF data. Note that the first dimenstion - !is the length of the character array, so need to start - !the dim_sizes allocation count at index two: - allocate(character(dim_sizes(1)) :: var(dim_sizes(2)), stat=errcode, errmsg=errmsg) + !read-in the NetCDF data. Note that the first dimension + !of dim_sizes is the length of the character variable, + !and is not included in the 'alloc_dims' array. + allocate(character(dim_sizes(1)) :: var(alloc_dims(1)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1700,11 +1774,21 @@ subroutine get_netcdf_var_char_2d(this, varname, var, errmsg, errcode, start, co return end if + !Check if variable subsetting is requested and valid: + !Note that this involves ignoring the first dimension, + !which is just the character length: + call var_subset_check(varname, var_ndims, dim_sizes(2:), do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and - !read-in the NetCDF data. Note that the first dimenstion - !is the length of the character array, so need to start - !the dim_sizes allocation count at index two: - allocate(character(dim_sizes(1)) :: var(dim_sizes(2), dim_sizes(3)), stat=errcode, errmsg=errmsg) + !read-in the NetCDF data. Note that the first dimension + !of dim_sizes is the length of the character variable, + !and is not included in the 'alloc_dims' array. + allocate(character(dim_sizes(1)) :: var(alloc_dims(1), alloc_dims(2)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1830,11 +1914,21 @@ subroutine get_netcdf_var_char_3d(this, varname, var, errmsg, errcode, start, co return end if + !Check if variable subsetting is requested and valid: + !Note that this involves ignoring the first dimension, + !which is just the character length: + call var_subset_check(varname, var_ndims, dim_sizes(2:), do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and - !read-in the NetCDF data. Note that the first dimenstion - !is the length of the character array, so need to start - !the dim_sizes allocation count at index two: - allocate(character(dim_sizes(1)) :: var(dim_sizes(2), dim_sizes(3), dim_sizes(4)), stat=errcode, errmsg=errmsg) + !read-in the NetCDF data. Note that the first dimension + !of dim_sizes is the length of the character variable, + !and is not included in the 'alloc_dims' array. + allocate(character(dim_sizes(1)) :: var(alloc_dims(1), alloc_dims(2), alloc_dims(3)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1958,11 +2052,21 @@ subroutine get_netcdf_var_char_4d(this, varname, var, errmsg, errcode, start, co return end if + !Check if variable subsetting is requested and valid + !Note that this involves ignoring the first dimension, + !which is just the character length: + call var_subset_check(varname, var_ndims, dim_sizes(2:), do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and - !read-in the NetCDF data. Note that the first dimenstion - !is the length of the character array, so need to start - !the dim_sizes allocation count at index two: - allocate(character(dim_sizes(1)) :: var(dim_sizes(2), dim_sizes(3), dim_sizes(4), dim_sizes(5)), & + !read-in the NetCDF data. Note that the first dimension + !of dim_sizes is the length of the character variable, + !and is not included in the 'alloc_dims' array. + allocate(character(dim_sizes(1)) :: var(alloc_dims(1), alloc_dims(2), alloc_dims(3), alloc_dims(4)), & stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: @@ -2087,11 +2191,21 @@ subroutine get_netcdf_var_char_5d(this, varname, var, errmsg, errcode, start, co return end if + !Check if variable subsetting is requested and valid + !Note that this involves ignoring the first dimension, + !which is just the character length: + call var_subset_check(varname, var_ndims, dim_sizes(2:), do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and - !read-in the NetCDF data. Note that the first dimenstion - !is the length of the character array, so need to start - !the dim_sizes allocation count at index two: - allocate(character(dim_sizes(1)) :: var(dim_sizes(2), dim_sizes(3), dim_sizes(4), dim_sizes(5), dim_sizes(6)), & + !read-in the NetCDF data. Note that the first dimension + !of dim_sizes is the length of the character variable, + !and is not included in the 'alloc_dims' array. + allocate(character(dim_sizes(1)) :: var(alloc_dims(1), alloc_dims(2), alloc_dims(3), alloc_dims(4), alloc_dims(5)), & stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index a4a05426..6f52ad8a 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1124,7 +1124,7 @@ subroutine test_pio_reader_int_get_var_bad_rank() ! Expected error message: character(len=*), parameter :: expected_err_msg = & - "Variable 'pressure' isn't declared with the correct number of dimensions" + "Variable 'pressure' isn't declared with the correct number of dimensions. Expected 1 dimensions, but is declared with 2 dimensions." ! Begin test: @@ -1173,7 +1173,8 @@ subroutine test_pio_reader_char_get_var_bad_rank() ! Expected error message: character(len=*), parameter :: expected_err_msg = & - "Variable 'gas_names' isn't declared with the correct number of dimensions" + "Variable 'gas_names' isn't declared with the correct number of dimensions. Expected 1 dimensions, but is declared with 2 dimensions." + ! Begin test: @@ -1224,7 +1225,8 @@ subroutine test_pio_reader_real_get_var_bad_rank() ! Expected error message: character(len=*), parameter :: expected_err_msg = & - "Variable 'press_ref' isn't declared with the correct number of dimensions" + "Variable 'press_ref' isn't declared with the correct number of dimensions. Expected 1 dimensions, but is declared with 2 dimensions." + ! Begin test: From 76fc32a1b31c90b1014dc7b7f29f1163dd9bd4bc Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 14 Aug 2025 15:44:20 -0600 Subject: [PATCH 037/150] Fix syntax error. --- src/physics/utils/pio_reader.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 8a136fd5..8c10ab26 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -750,7 +750,7 @@ subroutine get_netcdf_var_int_5d(this, varname, var, errmsg, errcode, start, cou !Now attempt to allocate and initialize variable, and !read-in the NetCDF data: - allocate(var(dalloc_dims(1), alloc_dims(2), alloc_dims(3), alloc_dims(4), alloc_dims(5)), & + allocate(var(alloc_dims(1), alloc_dims(2), alloc_dims(3), alloc_dims(4), alloc_dims(5)), & stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: From ee27eb3b6337eb38686c26ac8c13da86ae4ebc48 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 14 Aug 2025 16:07:52 -0600 Subject: [PATCH 038/150] Adjust 'var_ndims' for charactervariables. --- src/physics/utils/pio_reader.F90 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 8c10ab26..ef5234a2 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -1575,7 +1575,7 @@ subroutine get_netcdf_var_char_1d(this, varname, var, errmsg, errcode, start, co !Ideally the actual type would be checked and handled !differently, but for now just confirm it's a character !array and check for ndims = rank+1 - integer, parameter :: var_ndims = 2 !Number of expected dimensions for variable in NetCDF file + integer, parameter :: var_ndims = 1 !Number of expected dimensions for variable in NetCDF file logical :: do_subset !Will variable subsetting be done? Answer provided by !var_subset_check subroutine. @@ -1713,7 +1713,7 @@ subroutine get_netcdf_var_char_2d(this, varname, var, errmsg, errcode, start, co !Ideally the actual type would be checked and handled !differently, but for now just confirm it's a character !array and check for ndims = rank+1 - integer, parameter :: var_ndims = 3 !Number of expected dimensions for variable in NetCDF file + integer, parameter :: var_ndims = 2 !Number of expected dimensions for variable in NetCDF file logical :: do_subset !Will variable subsetting be done? Answer provided by !var_subset_check subroutine. @@ -1851,7 +1851,7 @@ subroutine get_netcdf_var_char_3d(this, varname, var, errmsg, errcode, start, co !Ideally the actual type would be checked and handled !differently, but for now just confirm it's a character !array and check for ndims = rank+1 - integer, parameter :: var_ndims = 4 !Number of expected dimensions for variable in NetCDF file + integer, parameter :: var_ndims = 3 !Number of expected dimensions for variable in NetCDF file logical :: do_subset !Will variable subsetting be done? Answer provided by !var_subset_check subroutine. @@ -1991,7 +1991,7 @@ subroutine get_netcdf_var_char_4d(this, varname, var, errmsg, errcode, start, co !Ideally the actual type would be checked and handled !differently, but for now just confirm it's a character !array and check for ndims = rank+1 - integer, parameter :: var_ndims = 5 !Number of expected dimensions for variable in NetCDF file + integer, parameter :: var_ndims = 4 !Number of expected dimensions for variable in NetCDF file logical :: do_subset !Will variable subsetting be done? Answer provided by !var_subset_check subroutine. @@ -2130,7 +2130,7 @@ subroutine get_netcdf_var_char_5d(this, varname, var, errmsg, errcode, start, co !Ideally the actual type would be checked and handled !differently, but for now just confirm it's a character !array and check for ndims = rank+1 - integer, parameter :: var_ndims = 6 !Number of expected dimensions for variable in NetCDF file + integer, parameter :: var_ndims = 5 !Number of expected dimensions for variable in NetCDF file logical :: do_subset !Will variable subsetting be done? Answer provided by !var_subset_check subroutine. From 0c8568817733ff2aa337a81881e245cc9d0c8ddc Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 15 Aug 2025 11:16:14 -0600 Subject: [PATCH 039/150] Remove extra space in expected error message. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 6f52ad8a..25437b13 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1124,7 +1124,7 @@ subroutine test_pio_reader_int_get_var_bad_rank() ! Expected error message: character(len=*), parameter :: expected_err_msg = & - "Variable 'pressure' isn't declared with the correct number of dimensions. Expected 1 dimensions, but is declared with 2 dimensions." + "Variable 'pressure' isn't declared with the correct number of dimensions. Expected 1 dimensions, but is declared with 2 dimensions." ! Begin test: @@ -1173,7 +1173,7 @@ subroutine test_pio_reader_char_get_var_bad_rank() ! Expected error message: character(len=*), parameter :: expected_err_msg = & - "Variable 'gas_names' isn't declared with the correct number of dimensions. Expected 1 dimensions, but is declared with 2 dimensions." + "Variable 'gas_names' isn't declared with the correct number of dimensions. Expected 1 dimensions, but is declared with 2 dimensions." ! Begin test: @@ -1225,7 +1225,7 @@ subroutine test_pio_reader_real_get_var_bad_rank() ! Expected error message: character(len=*), parameter :: expected_err_msg = & - "Variable 'press_ref' isn't declared with the correct number of dimensions. Expected 1 dimensions, but is declared with 2 dimensions." + "Variable 'press_ref' isn't declared with the correct number of dimensions. Expected 1 dimensions, but is declared with 2 dimensions." ! Begin test: From dbc7354745baa675e9bcbbbbe004218dff807846 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 15 Aug 2025 14:28:24 -0600 Subject: [PATCH 040/150] Change error message text, and add additional unit tests. --- src/physics/utils/pio_reader.F90 | 2 +- .../fortran/src/pio_reader/test_pio_reader.pf | 109 ++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index ef5234a2..8ce82337 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -2499,7 +2499,7 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims errcode = bad_subset_num_elem_err write(errmsg, '(3a,i0,a,i0,a)') & "The 'count' array for variable '", trim(varname), & - "' has too many elements with a count greater than one. Expected at most ", & + "' has too many elements with a value greater than one. Expected at most ", & var_ndims, " but got ", count_true_dim_num, "." return end if diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 25437b13..0d536854 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1385,6 +1385,63 @@ subroutine test_pio_reader_2d_int_full_subset_read() end subroutine test_pio_reader_2d_int_full_subset_read +!---------------------------------------- + +@test +subroutine test_pio_reader_1d_real_full_subset_read() + !Check that the PIO reader can read a 1D real variable + !from a file successfully with subsetting that has + !a count array with the same rank as the output variable. + + use funit + + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t + + class(abstract_netcdf_reader_t), pointer :: reader + + integer :: errcode + character(len=256) :: errmsg + + !Variable to be read: + real(kind_phys), allocatable :: press_ref(:) + + ! File name for testing: + ! NOTE: This specific file is added during the Github Actions workflow. + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + + ! Begin test: + + reader => create_netcdf_reader_t() + + ! Open file: + call reader%open_file(fname, errmsg, errcode) + + + ! Read 1D integer variable: + call reader%get_var("press_ref", press_ref, errmsg, errcode, start=(/10/), count=(/2/)) + + ! Check that the variable was read successfully: + @assertEqual('', errmsg) + @assertEqual(0, errcode) + + ! Check that the variable's properties are correct: + @assertTrue(allocated(press_ref)) + @assertEqual(2, size(press_ref)) + + ! Check that the variable's values are correct: + @assertEqual(18127.2241875, press_ref(1), tolerance=1.0e-7) + @assertEqual(14841.3159102, press_ref(2), tolerance=1.0e-7) + + ! Perform test cleanup: + deallocate(press_ref) + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) + +end subroutine test_pio_reader_1d_int_full_subset_read + !+++++++++++++++++++++++++++++++++++++++++++++++++++++ !PIO reader variable subsetting error tests !+++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -1715,6 +1772,58 @@ subroutine test_pio_reader_bad_count_element_range() end subroutine test_pio_reader_bad_count_element_range +!----------------------------------------- + +@test +subroutine test_pio_reader_bad_dim_reduce_elem_num() + ! Check that the PIO reader raises the correct error + ! when attempting to read a variable with a "count" + ! argument for a file variable with a larger + ! number of dimensions as the output variable, but + ! with the incorrect number of "non-one" elements. + + use funit + + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t + + use ccpp_kinds, only: kind_phys + + class(abstract_netcdf_reader_t), pointer :: reader + + integer :: errcode + character(len=256) :: errmsg + + ! Variable to be read: + real(kind_phys), allocatable :: press_ref(:) + + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + + ! Expected error message: + character(len=*), parameter :: expected_err_msg = & + "The 'count' array for variable 'press_ref' has too many elements with a value greater than one. Expected at most 1, but got 3." + + ! Begin test: + + reader => create_netcdf_reader_t() + + ! Open file: + call reader%open_file(fname, errmsg, errcode) + + ! Attempt to subset higher-dimensional variable with incorrect count elements: + call reader%get_var("vmr_ref", press_ref, errmsg, errcode, start=(/1,1,1/), count=(/14,20,2/)) + + ! Check that the correct error was raised: + @assertNotEqual(0, errcode) + @assertEqual(expected_err_msg, trim(errmsg)) + + ! Perform test cleanup: + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) + +end subroutine test_pio_reader_bad_dim_reduce_elem_num !++++++++++++++++++++++++++++++++++++++++++++++++++++ From ac6fce9f3cd6aaed01fde8320a5d3110e2c8b6f8 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 15 Aug 2025 20:12:07 -0600 Subject: [PATCH 041/150] Fix use statements in pFUnit Fortran code. --- .../unit/fortran/src/pio_reader/test_pio_reader.pf | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 0d536854..870b92b5 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1398,6 +1398,8 @@ subroutine test_pio_reader_1d_real_full_subset_read() use ccpp_io_reader, only: abstract_netcdf_reader_t use ccpp_io_reader, only: create_netcdf_reader_t + use ccpp_kinds, only: kind_phys + class(abstract_netcdf_reader_t), pointer :: reader integer :: errcode @@ -1458,8 +1460,6 @@ subroutine test_pio_reader_missing_start_argument() use ccpp_io_reader, only: abstract_netcdf_reader_t use ccpp_io_reader, only: create_netcdf_reader_t - use ccpp_kinds, only: kind_phys - class(abstract_netcdf_reader_t), pointer :: reader integer :: errcode @@ -1510,8 +1510,6 @@ subroutine test_pio_reader_missing_count_argument() use ccpp_io_reader, only: abstract_netcdf_reader_t use ccpp_io_reader, only: create_netcdf_reader_t - use ccpp_kinds, only: kind_phys - class(abstract_netcdf_reader_t), pointer :: reader integer :: errcode @@ -1561,8 +1559,6 @@ subroutine test_pio_reader_bad_num_start_elements() use ccpp_io_reader, only: abstract_netcdf_reader_t use ccpp_io_reader, only: create_netcdf_reader_t - use ccpp_kinds, only: kind_phys - class(abstract_netcdf_reader_t), pointer :: reader integer :: errcode @@ -1612,8 +1608,6 @@ subroutine test_pio_reader_bad_num_count_elements() use ccpp_io_reader, only: abstract_netcdf_reader_t use ccpp_io_reader, only: create_netcdf_reader_t - use ccpp_kinds, only: kind_phys - class(abstract_netcdf_reader_t), pointer :: reader integer :: errcode @@ -1663,8 +1657,6 @@ subroutine test_pio_reader_bad_start_element_range() use ccpp_io_reader, only: abstract_netcdf_reader_t use ccpp_io_reader, only: create_netcdf_reader_t - use ccpp_kinds, only: kind_phys - class(abstract_netcdf_reader_t), pointer :: reader integer :: errcode @@ -1724,8 +1716,6 @@ subroutine test_pio_reader_bad_count_element_range() use ccpp_io_reader, only: abstract_netcdf_reader_t use ccpp_io_reader, only: create_netcdf_reader_t - use ccpp_kinds, only: kind_phys - class(abstract_netcdf_reader_t), pointer :: reader integer :: errcode From 6f184f21b393e1f97deaeffb8ff34bc2b18d12f5 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 15 Aug 2025 20:27:11 -0600 Subject: [PATCH 042/150] Move count bounds checking outside of output var check, and fix test subroutine name. --- src/physics/utils/pio_reader.F90 | 29 ++++++++++--------- .../fortran/src/pio_reader/test_pio_reader.pf | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 8ce82337..87246dd4 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -2460,29 +2460,30 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims end if end do + !Check if count indices are within bounds: + do i = 1, file_var_dim_num + if ((count(i) < 1) .or. (count(i) > dim_sizes(i))) then + errcode = bad_subset_range_err + write(errmsg, '(a,i0,3a,i0,a,i0,a)') & + "Element ", i, " of 'count' for variable '", & + trim(varname), "' is out of bounds. Expected 1 to ", & + dim_sizes(i), " but got ", count(i), "." + return + end if + end do + !Check if count has the same number of elements as the output variable: if (size(count) == var_ndims) then - !If so, then check that count is positive and - !start + count is within bounds - !for each dimension: - do i = 1, file_var_dim_num - if ((count(i) < 1) .or. (count(i) > dim_sizes(i))) then - errcode = bad_subset_range_err - write(errmsg, '(a,i0,3a,i0,a,i0,a)') & - "Element ", i, " of 'count' for variable '", & - trim(varname), "' is out of bounds. Expected 1 to ", & - dim_sizes(i), " but got ", count(i), "." - return - end if - end do - !Start and Count are good, so notify caller that subsetting + !If so, then Start and Count are good, + !so notify caller that subsetting !will occur, and set alloc_dims to match 'count': do_subset = .true. allocate(alloc_dims, source=count, stat=errcode, errmsg=errmsg) return !Nothing more to do here. else if (size(count) > var_ndims) then + !The subsetting appears to be reducing !the dimensionality of the file variable. !Thus make sure that the number of dimensions diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 870b92b5..b9a52f60 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1442,7 +1442,7 @@ subroutine test_pio_reader_1d_real_full_subset_read() deallocate(reader) nullify(reader) -end subroutine test_pio_reader_1d_int_full_subset_read +end subroutine test_pio_reader_1d_real_full_subset_read !+++++++++++++++++++++++++++++++++++++++++++++++++++++ !PIO reader variable subsetting error tests From 73b75a34cbb69027e3a13b46940c4d20d6094027 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 15 Aug 2025 20:58:42 -0600 Subject: [PATCH 043/150] Comment out real var subsetting test. --- .../fortran/src/pio_reader/test_pio_reader.pf | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index b9a52f60..712c3748 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1387,62 +1387,62 @@ end subroutine test_pio_reader_2d_int_full_subset_read !---------------------------------------- -@test -subroutine test_pio_reader_1d_real_full_subset_read() +!@test +!subroutine test_pio_reader_1d_real_full_subset_read() !Check that the PIO reader can read a 1D real variable !from a file successfully with subsetting that has !a count array with the same rank as the output variable. - use funit +! use funit - use ccpp_io_reader, only: abstract_netcdf_reader_t - use ccpp_io_reader, only: create_netcdf_reader_t +! use ccpp_io_reader, only: abstract_netcdf_reader_t +! use ccpp_io_reader, only: create_netcdf_reader_t - use ccpp_kinds, only: kind_phys +! use ccpp_kinds, only: kind_phys - class(abstract_netcdf_reader_t), pointer :: reader +! class(abstract_netcdf_reader_t), pointer :: reader - integer :: errcode - character(len=256) :: errmsg +! integer :: errcode +! character(len=256) :: errmsg !Variable to be read: - real(kind_phys), allocatable :: press_ref(:) +! real(kind_phys), allocatable :: press_ref(:) ! File name for testing: ! NOTE: This specific file is added during the Github Actions workflow. - character(len=*), parameter :: fname = & - "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" +! character(len=*), parameter :: fname = & +! "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" ! Begin test: - reader => create_netcdf_reader_t() +! reader => create_netcdf_reader_t() ! Open file: - call reader%open_file(fname, errmsg, errcode) +! call reader%open_file(fname, errmsg, errcode) ! Read 1D integer variable: - call reader%get_var("press_ref", press_ref, errmsg, errcode, start=(/10/), count=(/2/)) +! call reader%get_var("press_ref", press_ref, errmsg, errcode, start=(/10/), count=(/2/)) ! Check that the variable was read successfully: - @assertEqual('', errmsg) - @assertEqual(0, errcode) +! @assertEqual('', errmsg) +! @assertEqual(0, errcode) ! Check that the variable's properties are correct: - @assertTrue(allocated(press_ref)) - @assertEqual(2, size(press_ref)) +! @assertTrue(allocated(press_ref)) +! @assertEqual(2, size(press_ref)) ! Check that the variable's values are correct: - @assertEqual(18127.2241875, press_ref(1), tolerance=1.0e-7) - @assertEqual(14841.3159102, press_ref(2), tolerance=1.0e-7) +! @assertEqual(18127.2241875, press_ref(1), tolerance=1.0e-7) +! @assertEqual(14841.3159102, press_ref(2), tolerance=1.0e-7) ! Perform test cleanup: - deallocate(press_ref) - call reader%close_file(errmsg, errcode) - deallocate(reader) - nullify(reader) +! deallocate(press_ref) +! call reader%close_file(errmsg, errcode) +! deallocate(reader) +! nullify(reader) -end subroutine test_pio_reader_1d_real_full_subset_read +!end subroutine test_pio_reader_1d_real_full_subset_read !+++++++++++++++++++++++++++++++++++++++++++++++++++++ !PIO reader variable subsetting error tests From 59775cc81c6b919d9c028f2e68df4b0ff7b96dfe Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 15 Aug 2025 21:17:10 -0600 Subject: [PATCH 044/150] Fix bad subsetting test, and partially uncomment 1D real subset test. --- .../fortran/src/pio_reader/test_pio_reader.pf | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 712c3748..0f08e183 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1387,35 +1387,35 @@ end subroutine test_pio_reader_2d_int_full_subset_read !---------------------------------------- -!@test -!subroutine test_pio_reader_1d_real_full_subset_read() +@test +subroutine test_pio_reader_1d_real_full_subset_read() !Check that the PIO reader can read a 1D real variable !from a file successfully with subsetting that has !a count array with the same rank as the output variable. -! use funit + use funit -! use ccpp_io_reader, only: abstract_netcdf_reader_t -! use ccpp_io_reader, only: create_netcdf_reader_t + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t -! use ccpp_kinds, only: kind_phys + use ccpp_kinds, only: kind_phys -! class(abstract_netcdf_reader_t), pointer :: reader + class(abstract_netcdf_reader_t), pointer :: reader -! integer :: errcode -! character(len=256) :: errmsg + integer :: errcode + character(len=256) :: errmsg !Variable to be read: -! real(kind_phys), allocatable :: press_ref(:) + real(kind_phys), allocatable :: press_ref(:) ! File name for testing: ! NOTE: This specific file is added during the Github Actions workflow. -! character(len=*), parameter :: fname = & -! "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" ! Begin test: -! reader => create_netcdf_reader_t() + reader => create_netcdf_reader_t() ! Open file: ! call reader%open_file(fname, errmsg, errcode) @@ -1438,9 +1438,9 @@ end subroutine test_pio_reader_2d_int_full_subset_read ! Perform test cleanup: ! deallocate(press_ref) -! call reader%close_file(errmsg, errcode) -! deallocate(reader) -! nullify(reader) + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) !end subroutine test_pio_reader_1d_real_full_subset_read @@ -1792,7 +1792,7 @@ subroutine test_pio_reader_bad_dim_reduce_elem_num() ! Expected error message: character(len=*), parameter :: expected_err_msg = & - "The 'count' array for variable 'press_ref' has too many elements with a value greater than one. Expected at most 1, but got 3." + "The 'count' array for variable 'vmr_ref' has too many elements with a value greater than one. Expected at most 1, but got 3." ! Begin test: @@ -1802,7 +1802,7 @@ subroutine test_pio_reader_bad_dim_reduce_elem_num() call reader%open_file(fname, errmsg, errcode) ! Attempt to subset higher-dimensional variable with incorrect count elements: - call reader%get_var("vmr_ref", press_ref, errmsg, errcode, start=(/1,1,1/), count=(/14,20,2/)) + call reader%get_var("vmr_ref", press_ref, errmsg, errcode, start=(/1,1,1/), count=(/2,20,14/)) ! Check that the correct error was raised: @assertNotEqual(0, errcode) From 5b331ba11584a9509f5904d86f240d9637194846 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 15 Aug 2025 21:24:11 -0600 Subject: [PATCH 045/150] Uncomment 'end subroutine' line. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 0f08e183..c8b04ad9 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1442,7 +1442,7 @@ subroutine test_pio_reader_1d_real_full_subset_read() deallocate(reader) nullify(reader) -!end subroutine test_pio_reader_1d_real_full_subset_read +end subroutine test_pio_reader_1d_real_full_subset_read !+++++++++++++++++++++++++++++++++++++++++++++++++++++ !PIO reader variable subsetting error tests From 6402882b9772da2f3d355557834caca8b4d18935 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 15 Aug 2025 21:40:39 -0600 Subject: [PATCH 046/150] Remove comma from expected error message, and try reading press_ref variable. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index c8b04ad9..4ff161cb 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1418,11 +1418,10 @@ subroutine test_pio_reader_1d_real_full_subset_read() reader => create_netcdf_reader_t() ! Open file: -! call reader%open_file(fname, errmsg, errcode) - + call reader%open_file(fname, errmsg, errcode) ! Read 1D integer variable: -! call reader%get_var("press_ref", press_ref, errmsg, errcode, start=(/10/), count=(/2/)) + call reader%get_var("press_ref", press_ref, errmsg, errcode) !, start=(/10/), count=(/2/)) ! Check that the variable was read successfully: ! @assertEqual('', errmsg) @@ -1437,7 +1436,7 @@ subroutine test_pio_reader_1d_real_full_subset_read() ! @assertEqual(14841.3159102, press_ref(2), tolerance=1.0e-7) ! Perform test cleanup: -! deallocate(press_ref) + deallocate(press_ref) call reader%close_file(errmsg, errcode) deallocate(reader) nullify(reader) @@ -1792,7 +1791,7 @@ subroutine test_pio_reader_bad_dim_reduce_elem_num() ! Expected error message: character(len=*), parameter :: expected_err_msg = & - "The 'count' array for variable 'vmr_ref' has too many elements with a value greater than one. Expected at most 1, but got 3." + "The 'count' array for variable 'vmr_ref' has too many elements with a value greater than one. Expected at most 1 but got 3." ! Begin test: From ef23f22257f311ef6d213968b75e890eca9f24bf Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 15 Aug 2025 21:50:27 -0600 Subject: [PATCH 047/150] Try 1D real subset test again. --- .../unit/fortran/src/pio_reader/test_pio_reader.pf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 4ff161cb..f7313a16 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1421,19 +1421,19 @@ subroutine test_pio_reader_1d_real_full_subset_read() call reader%open_file(fname, errmsg, errcode) ! Read 1D integer variable: - call reader%get_var("press_ref", press_ref, errmsg, errcode) !, start=(/10/), count=(/2/)) + call reader%get_var("press_ref", press_ref, errmsg, errcode, start=(/10/), count=(/2/)) ! Check that the variable was read successfully: -! @assertEqual('', errmsg) -! @assertEqual(0, errcode) + @assertEqual('', errmsg) + @assertEqual(0, errcode) ! Check that the variable's properties are correct: -! @assertTrue(allocated(press_ref)) -! @assertEqual(2, size(press_ref)) + @assertTrue(allocated(press_ref)) + @assertEqual(2, size(press_ref)) ! Check that the variable's values are correct: -! @assertEqual(18127.2241875, press_ref(1), tolerance=1.0e-7) -! @assertEqual(14841.3159102, press_ref(2), tolerance=1.0e-7) + @assertEqual(18127.2241875, press_ref(1), tolerance=1.0e-7) + @assertEqual(14841.3159102, press_ref(2), tolerance=1.0e-7) ! Perform test cleanup: deallocate(press_ref) From e829e94d8a3672b00a6accca6a4fde12bb6b76bd Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 18 Aug 2025 14:37:01 -0600 Subject: [PATCH 048/150] Add 'pio_get_var' subsetting to remaining multi-dim get_var interfaces. --- src/physics/utils/pio_reader.F90 | 130 +++++++++++++++++++++++++++---- 1 file changed, 117 insertions(+), 13 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 87246dd4..dd945805 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -549,7 +549,15 @@ subroutine get_netcdf_var_int_3d(this, varname, var, errmsg, errcode, start, cou return end if var(:,:,:) = huge(1) - errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:,:,:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -654,7 +662,15 @@ subroutine get_netcdf_var_int_4d(this, varname, var, errmsg, errcode, start, cou return end if var(:,:,:,:) = huge(1) - errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:,:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:,:,:,:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:,:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -758,7 +774,15 @@ subroutine get_netcdf_var_int_5d(this, varname, var, errmsg, errcode, start, cou return end if var(:,:,:,:,:) = huge(1) - errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:,:,:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:,:,:,:,:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:,:,:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -974,7 +998,15 @@ subroutine get_netcdf_var_real_1d(this, varname, var, errmsg, errcode, start, co return end if var(:) = huge(1._kind_phys) - errcode = pio_get_var(pio_file_handle, var_id, var(:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -1077,7 +1109,15 @@ subroutine get_netcdf_var_real_2d(this, varname, var, errmsg, errcode, start, co return end if var(:,:) = huge(1._kind_phys) - errcode = pio_get_var(pio_file_handle, var_id, var(:,:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:,:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:,:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -1180,7 +1220,15 @@ subroutine get_netcdf_var_real_3d(this, varname, var, errmsg, errcode, start, co return end if var(:,:,:) = huge(1._kind_phys) - errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:,:,:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -1283,7 +1331,15 @@ subroutine get_netcdf_var_real_4d(this, varname, var, errmsg, errcode, start, co return end if var(:,:,:,:) = huge(1._kind_phys) - errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:,:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:,:,:,:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:,:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -1387,7 +1443,15 @@ subroutine get_netcdf_var_real_5d(this, varname, var, errmsg, errcode, start, co return end if var(:,:,:,:,:) = huge(1._kind_phys) - errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:,:,:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:,:,:,:,:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:,:,:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -1657,7 +1721,15 @@ subroutine get_netcdf_var_char_1d(this, varname, var, errmsg, errcode, start, co return end if var(:) = 'UNSET' - errcode = pio_get_var(pio_file_handle, var_id, var(:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -1795,7 +1867,15 @@ subroutine get_netcdf_var_char_2d(this, varname, var, errmsg, errcode, start, co return end if var(:,:) = 'UNSET' - errcode = pio_get_var(pio_file_handle, var_id, var(:,:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:,:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:,:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -1935,7 +2015,15 @@ subroutine get_netcdf_var_char_3d(this, varname, var, errmsg, errcode, start, co return end if var(:,:,:) = 'UNSET' - errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:,:,:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -2074,7 +2162,15 @@ subroutine get_netcdf_var_char_4d(this, varname, var, errmsg, errcode, start, co return end if var(:,:,:,:) = 'UNSET' - errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:,:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:,:,:,:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:,:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: @@ -2213,7 +2309,15 @@ subroutine get_netcdf_var_char_5d(this, varname, var, errmsg, errcode, start, co return end if var(:,:,:,:,:) = 'UNSET' - errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:,:,:)) + + if (do_subset) then + !If subsetting is requested, then read only the specified + !subset of the variable: + errcode = pio_get_var(pio_file_handle, var_id, start, count, var(:,:,:,:,:)) + else + !Otherwise, read the entire variable: + errcode = pio_get_var(pio_file_handle, var_id, var(:,:,:,:,:)) + end if if (errcode /= PIO_NOERR) then !Extract error message from PIO: From 2776f7c82c23e32589c302d26a8b607dd44f99d7 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 18 Aug 2025 14:56:48 -0600 Subject: [PATCH 049/150] Print out press_ref values and change order. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index f7313a16..01fae8ab 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1432,7 +1432,9 @@ subroutine test_pio_reader_1d_real_full_subset_read() @assertEqual(2, size(press_ref)) ! Check that the variable's values are correct: - @assertEqual(18127.2241875, press_ref(1), tolerance=1.0e-7) + write(*,*) "press_ref values = ", press_ref + !@assertEqual(18127.2241875, press_ref(1), tolerance=1.0e-7) + @assertEqual(press_ref(1), 18127.2241875, tolerance=1.0e-7) @assertEqual(14841.3159102, press_ref(2), tolerance=1.0e-7) ! Perform test cleanup: From 574e71eebccfb727af0266d0040508c22f2bbe0c Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 18 Aug 2025 15:12:18 -0600 Subject: [PATCH 050/150] Change write statement to see if that fixes build issue? --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 01fae8ab..a5a3ea32 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1432,11 +1432,12 @@ subroutine test_pio_reader_1d_real_full_subset_read() @assertEqual(2, size(press_ref)) ! Check that the variable's values are correct: - write(*,*) "press_ref values = ", press_ref !@assertEqual(18127.2241875, press_ref(1), tolerance=1.0e-7) @assertEqual(press_ref(1), 18127.2241875, tolerance=1.0e-7) @assertEqual(14841.3159102, press_ref(2), tolerance=1.0e-7) + write(*,*) "press_ref values = ", press_ref(:) + ! Perform test cleanup: deallocate(press_ref) call reader%close_file(errmsg, errcode) From 4add51e3b32809c535c3c9e624ba81cb1a67b737 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 18 Aug 2025 15:19:32 -0600 Subject: [PATCH 051/150] Removed commented-out assert statement. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index a5a3ea32..3af04096 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1432,7 +1432,6 @@ subroutine test_pio_reader_1d_real_full_subset_read() @assertEqual(2, size(press_ref)) ! Check that the variable's values are correct: - !@assertEqual(18127.2241875, press_ref(1), tolerance=1.0e-7) @assertEqual(press_ref(1), 18127.2241875, tolerance=1.0e-7) @assertEqual(14841.3159102, press_ref(2), tolerance=1.0e-7) From dfa8143a81f0da25f421deec526e46429a7fe5ad Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 18 Aug 2025 15:24:22 -0600 Subject: [PATCH 052/150] Add back-in original 'assert' statement. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 3af04096..a29a2611 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1432,10 +1432,9 @@ subroutine test_pio_reader_1d_real_full_subset_read() @assertEqual(2, size(press_ref)) ! Check that the variable's values are correct: - @assertEqual(press_ref(1), 18127.2241875, tolerance=1.0e-7) - @assertEqual(14841.3159102, press_ref(2), tolerance=1.0e-7) - write(*,*) "press_ref values = ", press_ref(:) + @assertEqual(18127.2241875, press_ref(1), tolerance=1.0e-7) + @assertEqual(14841.3159102, press_ref(2), tolerance=1.0e-7) ! Perform test cleanup: deallocate(press_ref) From 23f6459c50883814c9cc762072e276216e9f653f Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 18 Aug 2025 16:30:31 -0600 Subject: [PATCH 053/150] Formally declare expected values in real 1D subset test. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index a29a2611..e5ab4f09 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1413,6 +1413,10 @@ subroutine test_pio_reader_1d_real_full_subset_read() character(len=*), parameter :: fname = & "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + ! Variable values to compare against: + real(kind_phys), parameter :: expected_press_ref_1 = 18127.2241875_kind_phys + real(kind_phys), parameter :: expected_press_ref_2 = 14841.3159102_kind_phys + ! Begin test: reader => create_netcdf_reader_t() @@ -1432,9 +1436,8 @@ subroutine test_pio_reader_1d_real_full_subset_read() @assertEqual(2, size(press_ref)) ! Check that the variable's values are correct: - write(*,*) "press_ref values = ", press_ref(:) - @assertEqual(18127.2241875, press_ref(1), tolerance=1.0e-7) - @assertEqual(14841.3159102, press_ref(2), tolerance=1.0e-7) + @assertEqual(expected_press_ref_1, press_ref(1), tolerance=1.0e-7) + @assertEqual(expected_press_ref_2, press_ref(2), tolerance=1.0e-7) ! Perform test cleanup: deallocate(press_ref) From 72844d03dcaba3b6a8c51d026d45bba3aa529932 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 19 Aug 2025 09:36:15 -0600 Subject: [PATCH 054/150] Add ability to reduce output variable dimensionality via subsetting. Also add unit test. --- src/physics/utils/pio_reader.F90 | 15 +++++ .../fortran/src/pio_reader/test_pio_reader.pf | 61 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index dd945805..e2258179 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -2609,6 +2609,21 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims return end if + !Number of "true" dimensions in count is correct, so let's + !allocate alloc_dims to match the "count" array ignoring dimensions + !with a value of 1: + allocate(alloc_dims(count_true_dim_num), stat=errcode, errmsg=errmsg) + if(errcode /= 0) then + return !Error allocating alloc_dims, so return. + end if + + do i=1, size(count) + if (count(i) > 1) then + alloc_dims(i) = count(i) + end if + end do + return !Nothing more to do here. + else if (size(count) < var_ndims) then !I don't think this should ever happen as !we have already checked that count matches diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index e5ab4f09..f796cc74 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1447,6 +1447,67 @@ subroutine test_pio_reader_1d_real_full_subset_read() end subroutine test_pio_reader_1d_real_full_subset_read +!---------------------------------------- + +@test +subroutine test_pio_reader_1d_real_dim_reduce_subset_read + !Check that the PIO reader can read a 2D real variable + !from a file and subset the data into a 1D output variable. + + use funit + + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t + + use ccpp_kinds, only: kind_phys + + class(abstract_netcdf_reader_t), pointer :: reader + + integer :: errcode + character(len=256) :: errmsg + + !Variable to be read: + real(kind_phys), allocatable :: vmr_ref_1D(:) + + ! File name for testing: + ! NOTE: This specific file is added during the Github Actions workflow. + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + + ! Variable values to compare against: + real(kind_phys), parameter :: expected_vmr_ref_1 = 1._kind_phys + real(kind_phys), parameter :: expected_vmr_ref_2 = 1._kind_phys + + ! Begin test: + + reader => create_netcdf_reader_t() + + ! Open file: + call reader%open_file(fname, errmsg, errcode) + + ! Read 1D integer variable: + call reader%get_var("vmr_ref", vmr_ref_1D, errmsg, errcode, start=(/1,1,1/), count=(/2,1,1/)) + + ! Check that the variable was read successfully: + @assertEqual('', errmsg) + @assertEqual(0, errcode) + + ! Check that the variable's properties are correct: + @assertTrue(allocated(vmr_ref_1D)) + @assertEqual(2, size(vmr_ref_1D)) + + ! Check that the variable's values are correct: + @assertEqual(expected_vmr_ref_1, vmr_ref_1D(1), tolerance=1.0e-7) + @assertEqual(expected_vmr_ref_2, vmr_ref_1D(2), tolerance=1.0e-7) + + ! Perform test cleanup: + deallocate(vmr_ref_1D) + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) + +end subroutine test_pio_reader_1d_real_dim_reduce_subset_read + !+++++++++++++++++++++++++++++++++++++++++++++++++++++ !PIO reader variable subsetting error tests !+++++++++++++++++++++++++++++++++++++++++++++++++++++ From a20f5929be994d4d34b54e5209dc1794f56a3b8d Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 19 Aug 2025 10:32:24 -0600 Subject: [PATCH 055/150] Make output variable 3D to see if that works. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index f796cc74..7808c24b 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1467,7 +1467,7 @@ subroutine test_pio_reader_1d_real_dim_reduce_subset_read character(len=256) :: errmsg !Variable to be read: - real(kind_phys), allocatable :: vmr_ref_1D(:) + real(kind_phys), allocatable :: vmr_ref_1D(:,:,:) ! File name for testing: ! NOTE: This specific file is added during the Github Actions workflow. @@ -1497,8 +1497,8 @@ subroutine test_pio_reader_1d_real_dim_reduce_subset_read @assertEqual(2, size(vmr_ref_1D)) ! Check that the variable's values are correct: - @assertEqual(expected_vmr_ref_1, vmr_ref_1D(1), tolerance=1.0e-7) - @assertEqual(expected_vmr_ref_2, vmr_ref_1D(2), tolerance=1.0e-7) + @assertEqual(expected_vmr_ref_1, vmr_ref_1D(1,1,1), tolerance=1.0e-7) + @assertEqual(expected_vmr_ref_2, vmr_ref_1D(2,1,1), tolerance=1.0e-7) ! Perform test cleanup: deallocate(vmr_ref_1D) From 4f91699458436c251e927f53c0920384d1571558 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 22 Aug 2025 08:49:51 -0600 Subject: [PATCH 056/150] Remove variable dimension reduction functionality (it doesn't appear to easily work with PIO). --- src/physics/utils/pio_reader.F90 | 90 ++++--------------- .../fortran/src/pio_reader/test_pio_reader.pf | 78 +++------------- 2 files changed, 30 insertions(+), 138 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index e2258179..c0dfef51 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -2498,23 +2498,21 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims !Determin the total number of dimensions for variable on file: file_var_dim_num = size(dim_sizes) + !Check that the variable rank as specified by the caller + !matches what is found on the NetCDF file: + if(file_var_dim_num /= var_ndims) then + errcode = bad_var_rank_err + write(errmsg, '(4a,i0,a,i0,a)') & + "Variable '",trim(varname),"' isn't declared with the correct number of dimensions.", & + " Expected ", file_var_dim_num, " dimensions, but is declared with ", & + var_ndims, " dimensions." + return + end if + !If both start and count are not present, - !then check for matching ranks, and set alloc_dims - !to have the same dimensionality as the file variable: + !then set alloc_dims to have the same + !dimensionality as the file variable: if (.not. present(start) .and. .not. present(count)) then - - !Check that the variable rank as specified by the caller - !matches what is found on the NetCDF file: - if(file_var_dim_num /= var_ndims) then - errcode = bad_var_rank_err - write(errmsg, '(4a,i0,a,i0,a)') & - "Variable '",trim(varname),"' isn't declared with the correct number of dimensions.", & - " Expected ", file_var_dim_num, " dimensions, but is declared with ", & - var_ndims, " dimensions." - return - end if - - !Allocate "alloc_dims" to exactly match the file variable dimensions: allocate(alloc_dims, source=dim_sizes, stat=errcode, errmsg=errmsg) return !Nothing more to do here. end if @@ -2576,63 +2574,11 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims end if end do - !Check if count has the same number of elements as the output variable: - if (size(count) == var_ndims) then - - !If so, then Start and Count are good, - !so notify caller that subsetting - !will occur, and set alloc_dims to match 'count': - do_subset = .true. - allocate(alloc_dims, source=count, stat=errcode, errmsg=errmsg) - return !Nothing more to do here. - - else if (size(count) > var_ndims) then - - !The subsetting appears to be reducing - !the dimensionality of the file variable. - !Thus make sure that the number of dimensions - !with more than a count of "1" matches - !the output variable's dimensionality: - count_true_dim_num = 0 - do i=1, size(count) - if (count(i) > 1) then - count_true_dim_num = count_true_dim_num + 1 - end if - end do - - if (count_true_dim_num > var_ndims) then - errcode = bad_subset_num_elem_err - write(errmsg, '(3a,i0,a,i0,a)') & - "The 'count' array for variable '", trim(varname), & - "' has too many elements with a value greater than one. Expected at most ", & - var_ndims, " but got ", count_true_dim_num, "." - return - end if - - !Number of "true" dimensions in count is correct, so let's - !allocate alloc_dims to match the "count" array ignoring dimensions - !with a value of 1: - allocate(alloc_dims(count_true_dim_num), stat=errcode, errmsg=errmsg) - if(errcode /= 0) then - return !Error allocating alloc_dims, so return. - end if - - do i=1, size(count) - if (count(i) > 1) then - alloc_dims(i) = count(i) - end if - end do - return !Nothing more to do here. - - else if (size(count) < var_ndims) then - !I don't think this should ever happen as - !we have already checked that count matches - !file_var_dim num, but just in case let's - !throw an error here: - errcode = bad_subset_num_elem_err - errmsg = "The number of elements in the 'count' array for variable '"//trim(varname)//"' has too few dimensions." - return - end if + !If so, then Start and Count are good, + !so notify caller that subsetting + !will occur, and set alloc_dims to match 'count': + do_subset = .true. + allocate(alloc_dims, source=count, stat=errcode, errmsg=errmsg) end subroutine var_subset_check diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 7808c24b..502b9800 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1390,8 +1390,7 @@ end subroutine test_pio_reader_2d_int_full_subset_read @test subroutine test_pio_reader_1d_real_full_subset_read() !Check that the PIO reader can read a 1D real variable - !from a file successfully with subsetting that has - !a count array with the same rank as the output variable. + !from a file successfully with subsetting. use funit @@ -1450,9 +1449,9 @@ end subroutine test_pio_reader_1d_real_full_subset_read !---------------------------------------- @test -subroutine test_pio_reader_1d_real_dim_reduce_subset_read - !Check that the PIO reader can read a 2D real variable - !from a file and subset the data into a 1D output variable. +subroutine test_pio_reader_3d_real_full_subset_read + !Check that the PIO reader can read a 3D real variable + !from a file successfully with subsetting. use funit @@ -1467,7 +1466,7 @@ subroutine test_pio_reader_1d_real_dim_reduce_subset_read character(len=256) :: errmsg !Variable to be read: - real(kind_phys), allocatable :: vmr_ref_1D(:,:,:) + real(kind_phys), allocatable :: vmr_ref(:,:,:) ! File name for testing: ! NOTE: This specific file is added during the Github Actions workflow. @@ -1486,27 +1485,27 @@ subroutine test_pio_reader_1d_real_dim_reduce_subset_read call reader%open_file(fname, errmsg, errcode) ! Read 1D integer variable: - call reader%get_var("vmr_ref", vmr_ref_1D, errmsg, errcode, start=(/1,1,1/), count=(/2,1,1/)) + call reader%get_var("vmr_ref", vmr_ref, errmsg, errcode, start=(/1,1,1/), count=(/2,1,1/)) ! Check that the variable was read successfully: @assertEqual('', errmsg) @assertEqual(0, errcode) ! Check that the variable's properties are correct: - @assertTrue(allocated(vmr_ref_1D)) - @assertEqual(2, size(vmr_ref_1D)) + @assertTrue(allocated(vmr_ref)) + @assertEqual(2, size(vmr_ref)) ! Check that the variable's values are correct: - @assertEqual(expected_vmr_ref_1, vmr_ref_1D(1,1,1), tolerance=1.0e-7) - @assertEqual(expected_vmr_ref_2, vmr_ref_1D(2,1,1), tolerance=1.0e-7) + @assertEqual(expected_vmr_ref_1, vmr_ref(1,1,1), tolerance=1.0e-7) + @assertEqual(expected_vmr_ref_2, vmr_ref(2,1,1), tolerance=1.0e-7) ! Perform test cleanup: - deallocate(vmr_ref_1D) + deallocate(vmr_ref) call reader%close_file(errmsg, errcode) deallocate(reader) nullify(reader) -end subroutine test_pio_reader_1d_real_dim_reduce_subset_read +end subroutine test_pio_reader_3d_real_full_subset_read !+++++++++++++++++++++++++++++++++++++++++++++++++++++ !PIO reader variable subsetting error tests @@ -1826,59 +1825,6 @@ subroutine test_pio_reader_bad_count_element_range() end subroutine test_pio_reader_bad_count_element_range -!----------------------------------------- - -@test -subroutine test_pio_reader_bad_dim_reduce_elem_num() - ! Check that the PIO reader raises the correct error - ! when attempting to read a variable with a "count" - ! argument for a file variable with a larger - ! number of dimensions as the output variable, but - ! with the incorrect number of "non-one" elements. - - use funit - - use ccpp_io_reader, only: abstract_netcdf_reader_t - use ccpp_io_reader, only: create_netcdf_reader_t - - use ccpp_kinds, only: kind_phys - - class(abstract_netcdf_reader_t), pointer :: reader - - integer :: errcode - character(len=256) :: errmsg - - ! Variable to be read: - real(kind_phys), allocatable :: press_ref(:) - - character(len=*), parameter :: fname = & - "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" - - ! Expected error message: - character(len=*), parameter :: expected_err_msg = & - "The 'count' array for variable 'vmr_ref' has too many elements with a value greater than one. Expected at most 1 but got 3." - - ! Begin test: - - reader => create_netcdf_reader_t() - - ! Open file: - call reader%open_file(fname, errmsg, errcode) - - ! Attempt to subset higher-dimensional variable with incorrect count elements: - call reader%get_var("vmr_ref", press_ref, errmsg, errcode, start=(/1,1,1/), count=(/2,20,14/)) - - ! Check that the correct error was raised: - @assertNotEqual(0, errcode) - @assertEqual(expected_err_msg, trim(errmsg)) - - ! Perform test cleanup: - call reader%close_file(errmsg, errcode) - deallocate(reader) - nullify(reader) - -end subroutine test_pio_reader_bad_dim_reduce_elem_num - !++++++++++++++++++++++++++++++++++++++++++++++++++++ @test From c2ff44c979fa6ee055df5d47e742e00409cf5f18 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 22 Aug 2025 09:42:05 -0600 Subject: [PATCH 057/150] Add error if user tries to subset a scalar variable, along with a relevant unit test. --- src/physics/utils/pio_reader.F90 | 42 +++++++++++++-- .../fortran/src/pio_reader/test_pio_reader.pf | 53 +++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index c0dfef51..2b3db7a3 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -215,6 +215,15 @@ subroutine get_netcdf_var_int_0d(this, varname, var, errmsg, errcode, start, cou return end if + !Make sure the caller isn't trying to subset a scalar variable: + if (present(start) .or. present(count)) then + errcode = bad_subset_num_elem_err + errmsg = "Variable '"//trim(varname)//"' is a scalar variable, so start and count arguments must not be provided." + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and !read-in the NetCDF data: allocate(var, stat=errcode, errmsg=errmsg) @@ -886,6 +895,15 @@ subroutine get_netcdf_var_real_0d(this, varname, var, errmsg, errcode, start, co return end if + !Make sure the caller isn't trying to subset a scalar variable: + if (present(start) .or. present(count)) then + errcode = bad_subset_num_elem_err + errmsg = "Variable '"//trim(varname)//"' is a scalar variable, so start and count arguments must not be provided." + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and !read-in the NetCDF data: allocate(var, stat=errcode, errmsg=errmsg) @@ -1572,10 +1590,28 @@ subroutine get_netcdf_var_char_0d(this, varname, var, errmsg, errcode, start, co return end if + !Check that the variable rank as specified by the caller + !matches what is found on the NetCDF file: + if(size(dim_sizes) /= 1) then + errcode = bad_var_rank_err + errmsg = "Variable '"//trim(varname)//"' isn't declared with the correct number of dimensions" + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + + !Make sure the caller isn't trying to subset a scalar variable: + if (present(start) .or. present(count)) then + errcode = bad_subset_num_elem_err + errmsg = "Variable '"//trim(varname)//"' is a scalar variable, so start and count arguments must not be provided." + !Reset PIO back to original error handling method: + call pio_seterrorhandling(pio_file_handle, err_handling) + return + end if + !Now attempt to allocate and initialize variable, and - !read-in the NetCDF data. Note that the first dimenstion - !is the length of the character array, so need to start - !the dim_sizes allocation count at index two: + !read-in the NetCDF data. Note that the given dimension + !is the length of the character array for a scalar variable: allocate(character(dim_sizes(1)) :: var, stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 502b9800..0a284896 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1609,6 +1609,59 @@ subroutine test_pio_reader_missing_count_argument() end subroutine test_pio_reader_missing_count_argument +!----------------------------------------- + +@test +subroutine test_pio_reader_0d_real_subset_err() + ! Check that the PIO reader can raise the + ! correct error when a user attempts to subset + ! a scalar (0-D) variable. + + use funit + + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t + + use ccpp_kinds, only: kind_phys + + class(abstract_netcdf_reader_t), pointer :: reader + + integer :: errcode + character(len=256) :: errmsg + + ! Variable to be read: + real(kind_phys), allocatable :: press_ref_trop + + ! File name for testing: + ! NOTE: This specific file is added during the Github Actions workflow. + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + + ! Expected error message: + character(len=*), parameter :: expected_err_msg = & + "Variable 'press_ref_trop' is a scalar variable, so start and count arguments must not be provided." + + ! Begin test: + + reader => create_netcdf_reader_t() + + ! Open file: + call reader%open_file(fname, errmsg, errcode) + + ! Read 0D real variable: + call reader%get_var("press_ref_trop", press_ref_trop, errmsg, errcode, start=(/1/), count=(/1/)) + + ! Check that the correct error was raised: + @assertNotEqual(0, errcode) + @assertEqual(expected_err_msg, trim(errmsg)) + + ! Perform test cleanup: + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) + +end subroutine test_pio_reader_0d_real_subset_err + !----------------------------------------- @test From 77d876dca70dfe4c3ca4445bf0ea58df27a12800 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 22 Aug 2025 12:57:00 -0600 Subject: [PATCH 058/150] Add character variable subsetting unit test. --- .../fortran/src/pio_reader/test_pio_reader.pf | 81 ++++++++++++++++--- 1 file changed, 71 insertions(+), 10 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 0a284896..a572c46b 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1255,9 +1255,9 @@ end subroutine test_pio_reader_real_get_var_bad_rank @test subroutine test_pio_reader_1d_int_full_subset_read() - !Check that the PIO reader can read a 1D integer variable - !from a file successfully with subsetting that has - !a count array with the same rank as the output variable. + ! Check that the PIO reader can read a 1D integer variable + ! from a file successfully with subsetting that has + ! a count array with the same rank as the output variable. use funit @@ -1314,8 +1314,8 @@ end subroutine test_pio_reader_1d_int_full_subset_read @test subroutine test_pio_reader_2d_int_full_subset_read() - !Check that the PIO reader can read a 2D integer variable - !from a file successfully. + ! Check that the PIO reader can read a 2D integer variable + ! from a file successfully. use funit @@ -1389,8 +1389,8 @@ end subroutine test_pio_reader_2d_int_full_subset_read @test subroutine test_pio_reader_1d_real_full_subset_read() - !Check that the PIO reader can read a 1D real variable - !from a file successfully with subsetting. + ! Check that the PIO reader can read a 1D real variable + ! from a file successfully with subsetting. use funit @@ -1450,8 +1450,8 @@ end subroutine test_pio_reader_1d_real_full_subset_read @test subroutine test_pio_reader_3d_real_full_subset_read - !Check that the PIO reader can read a 3D real variable - !from a file successfully with subsetting. + ! Check that the PIO reader can read a 3D real variable + ! from a file successfully with subsetting. use funit @@ -1465,7 +1465,7 @@ subroutine test_pio_reader_3d_real_full_subset_read integer :: errcode character(len=256) :: errmsg - !Variable to be read: + ! Variable to be read: real(kind_phys), allocatable :: vmr_ref(:,:,:) ! File name for testing: @@ -1507,6 +1507,67 @@ subroutine test_pio_reader_3d_real_full_subset_read end subroutine test_pio_reader_3d_real_full_subset_read +!---------------------------------------- + +@test +subroutine test_pio_reader_1d_char_full_subset_read() + ! Check that the PIO reader can read a 3D real + ! variable from a file successfully with subsetting. + + use funit + + use ccpp_io_reader, only: abstract_netcdf_reader_t + use ccpp_io_reader, only: create_netcdf_reader_t + + class(abstract_netcdf_reader_t), pointer :: reader + + integer :: errcode + character(len=256) :: errmsg + + ! Variable to be read: + character(len=:), allocatable :: gas_names(:) + + ! File name for testing: + ! NOTE: This specific file is added during the Github Actions workflow. + character(len=*), parameter :: fname = & + "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + + ! Begin test: + + reader => create_netcdf_reader_t() + + ! Open file: + call reader%open_file(fname, errmsg, errcode) + + ! Read 1D character variable: + call reader%get_var("gas_names", gas_names, errmsg, errcode, start=(/10/), count=(/6/)) + + ! Check that the variable was read successfully: + @assertEqual('', errmsg) + @assertEqual(0, errcode) + + ! Check that the variable's properties are correct: + @assertTrue(allocated(gas_names)) + @assertEqual(32, len(gas_names)) + @assertEqual(19, size(gas_names)) + + ! Check that the variable's values are correct: + @assertEqual("cfc11", trim(gas_names(10))) + @assertEqual("cfc12", trim(gas_names(11))) + @assertEqual("cfc22", trim(gas_names(12))) + @assertEqual("hfc143a", trim(gas_names(13))) + @assertEqual("hfc125", trim(gas_names(14))) + @assertEqual("hfc23", trim(gas_names(15))) + + + ! Perform test cleanup: + deallocate(gas_names) + call reader%close_file(errmsg, errcode) + deallocate(reader) + nullify(reader) + +end subroutine test_pio_reader_1d_char_read + !+++++++++++++++++++++++++++++++++++++++++++++++++++++ !PIO reader variable subsetting error tests !+++++++++++++++++++++++++++++++++++++++++++++++++++++ From 1625101f4d68a53c1bf618552b0fa56ccc12ed6e Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 22 Aug 2025 12:58:42 -0600 Subject: [PATCH 059/150] Fix character size assert check in subsetting test. --- .../fortran/src/pio_reader/test_pio_reader.pf | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index a572c46b..3add78df 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -80,8 +80,8 @@ end subroutine test_pio_reader_open_close_file @test subroutine test_pio_reader_already_opened_file_err() - !Check that the PIO reader raises the correct error - !when attempting to open a file that is already opened. + ! Check that the PIO reader raises the correct error + ! when attempting to open a file that is already opened. use funit @@ -127,8 +127,8 @@ end subroutine test_pio_reader_already_opened_file_err @test subroutine test_pio_reader_no_file_err() - !Check that the PIO reader raises the correct error - !when attempting to open a file that does not exist. + ! Check that the PIO reader raises the correct error + ! when attempting to open a file that does not exist. use funit @@ -174,8 +174,8 @@ end subroutine test_pio_reader_no_file_err @test subroutine test_pio_reader_1d_int_read() - !Check that the PIO reader can read a 1D integer variable - !from a file successfully. + ! Check that the PIO reader can read a 1D integer variable + ! from a file successfully. use funit @@ -230,8 +230,8 @@ end subroutine test_pio_reader_1d_int_read @test subroutine test_pio_reader_2d_int_read() - !Check that the PIO reader can read a 2D integer variable - !from a file successfully. + ! Check that the PIO reader can read a 2D integer variable + ! from a file successfully. use funit @@ -660,10 +660,10 @@ end subroutine test_pio_reader_3d_real_read !---------------------------------------- @disable -!Reading a 4D real variable works in a normal -!CAM-SIMA build, but not in this pFUnit build (at least for GCCv13). -!Thus going ahead and leaving this disabled for now -!until more experimentation can be done. +! Reading a 4D real variable works in a normal +! CAM-SIMA build, but not in this pFUnit build (at least for GCCv13). +! Thus going ahead and leaving this disabled for now +! until more experimentation can be done. subroutine test_pio_reader_4d_real_read_with_dims() ! Check that the PIO reader can read a 4D real variable ! from a file successfully, and that the dimensions are correct. @@ -1549,7 +1549,7 @@ subroutine test_pio_reader_1d_char_full_subset_read() ! Check that the variable's properties are correct: @assertTrue(allocated(gas_names)) @assertEqual(32, len(gas_names)) - @assertEqual(19, size(gas_names)) + @assertEqual(6, size(gas_names)) ! Check that the variable's values are correct: @assertEqual("cfc11", trim(gas_names(10))) From da268b0a2602193e3882fa7ee88f97812f2d1730 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 22 Aug 2025 13:03:45 -0600 Subject: [PATCH 060/150] Fix syntax error in subset test. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 3add78df..9f3a38bd 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1566,7 +1566,7 @@ subroutine test_pio_reader_1d_char_full_subset_read() deallocate(reader) nullify(reader) -end subroutine test_pio_reader_1d_char_read +end subroutine test_pio_reader_1d_char_full_subset_read !+++++++++++++++++++++++++++++++++++++++++++++++++++++ !PIO reader variable subsetting error tests From 37f1dd55734a2f682b7f925d6d0961deaa7fd1ff Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 22 Aug 2025 13:34:43 -0600 Subject: [PATCH 061/150] Add debugging output. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 9f3a38bd..bd4813b4 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1532,6 +1532,8 @@ subroutine test_pio_reader_1d_char_full_subset_read() character(len=*), parameter :: fname = & "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" + integer :: i !DEBUGGING -JN + ! Begin test: reader => create_netcdf_reader_t() @@ -1551,6 +1553,11 @@ subroutine test_pio_reader_1d_char_full_subset_read() @assertEqual(32, len(gas_names)) @assertEqual(6, size(gas_names)) + ! DEBUGGING -JN + do i=1, size(gas_names) + write(*,*) "gas_names(", i, ") = '", trim(gas_names(i)), "'" + end do + ! Check that the variable's values are correct: @assertEqual("cfc11", trim(gas_names(10))) @assertEqual("cfc12", trim(gas_names(11))) From 9cee38dcf1c4311ad6f1092c29d3dda54bb449c8 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 22 Aug 2025 13:54:25 -0600 Subject: [PATCH 062/150] Try subsetting with character array length provided. --- src/physics/utils/pio_reader.F90 | 8 ++++++-- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 2b3db7a3..56076419 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -1739,7 +1739,9 @@ subroutine get_netcdf_var_char_1d(this, varname, var, errmsg, errcode, start, co !Check if variable subsetting is requested and valid !Note that this involves ignoring the first dimension, !which is just the character length: - call var_subset_check(varname, var_ndims, dim_sizes(2:), do_subset, alloc_dims, errmsg, errcode, start, count) + !call var_subset_check(varname, var_ndims, dim_sizes(2:), do_subset, alloc_dims, errmsg, errcode, start, count) + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) + if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1750,7 +1752,9 @@ subroutine get_netcdf_var_char_1d(this, varname, var, errmsg, errcode, start, co !read-in the NetCDF data. Note that the first dimension !of dim_sizes is the length of the character variable, !and is not included in the 'alloc_dims' array. - allocate(character(dim_sizes(1)) :: var(alloc_dims(1)), stat=errcode, errmsg=errmsg) + allocate(character(alloc_dims(1)) :: var(alloc_dims(2)), stat=errcode, errmsg=errmsg) + !allocate(character(dim_sizes(1)) :: var(alloc_dims(1)), stat=errcode, errmsg=errmsg) + if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index bd4813b4..78360fed 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1542,7 +1542,10 @@ subroutine test_pio_reader_1d_char_full_subset_read() call reader%open_file(fname, errmsg, errcode) ! Read 1D character variable: - call reader%get_var("gas_names", gas_names, errmsg, errcode, start=(/10/), count=(/6/)) + ! NOTE: character variable subsetting requires passing in + ! the string Length as the first dimension of the start and + ! count arrays. + call reader%get_var("gas_names", gas_names, errmsg, errcode, start=(/1,10/), count=(/32,6/)) ! Check that the variable was read successfully: @assertEqual('', errmsg) From 172871276ddeffdebfa0906bb2c1313da2234c57 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 22 Aug 2025 14:03:17 -0600 Subject: [PATCH 063/150] Increase 'var_ndims' by one to avoid incorrect error. --- src/physics/utils/pio_reader.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 56076419..9a42e96b 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -1675,7 +1675,7 @@ subroutine get_netcdf_var_char_1d(this, varname, var, errmsg, errcode, start, co !Ideally the actual type would be checked and handled !differently, but for now just confirm it's a character !array and check for ndims = rank+1 - integer, parameter :: var_ndims = 1 !Number of expected dimensions for variable in NetCDF file + integer, parameter :: var_ndims = 2 !Number of expected dimensions for variable in NetCDF file logical :: do_subset !Will variable subsetting be done? Answer provided by !var_subset_check subroutine. From dc0e3dc2930c3c0c01a6c3d9d8183346697ea5d2 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 22 Aug 2025 14:25:40 -0600 Subject: [PATCH 064/150] Fix unit test assert statements, and update remaining character get_var interfactes. --- src/physics/utils/pio_reader.F90 | 30 ++++++++----------- .../fortran/src/pio_reader/test_pio_reader.pf | 12 ++++---- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 9a42e96b..4bcfc58c 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -1592,7 +1592,7 @@ subroutine get_netcdf_var_char_0d(this, varname, var, errmsg, errcode, start, co !Check that the variable rank as specified by the caller !matches what is found on the NetCDF file: - if(size(dim_sizes) /= 1) then + if(size(dim_sizes) /= var_ndims) then errcode = bad_var_rank_err errmsg = "Variable '"//trim(varname)//"' isn't declared with the correct number of dimensions" !Reset PIO back to original error handling method: @@ -1739,7 +1739,6 @@ subroutine get_netcdf_var_char_1d(this, varname, var, errmsg, errcode, start, co !Check if variable subsetting is requested and valid !Note that this involves ignoring the first dimension, !which is just the character length: - !call var_subset_check(varname, var_ndims, dim_sizes(2:), do_subset, alloc_dims, errmsg, errcode, start, count) call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) if (errcode /= 0) then @@ -1753,7 +1752,6 @@ subroutine get_netcdf_var_char_1d(this, varname, var, errmsg, errcode, start, co !of dim_sizes is the length of the character variable, !and is not included in the 'alloc_dims' array. allocate(character(alloc_dims(1)) :: var(alloc_dims(2)), stat=errcode, errmsg=errmsg) - !allocate(character(dim_sizes(1)) :: var(alloc_dims(1)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: @@ -1825,7 +1823,7 @@ subroutine get_netcdf_var_char_2d(this, varname, var, errmsg, errcode, start, co !Ideally the actual type would be checked and handled !differently, but for now just confirm it's a character !array and check for ndims = rank+1 - integer, parameter :: var_ndims = 2 !Number of expected dimensions for variable in NetCDF file + integer, parameter :: var_ndims = 3 !Number of expected dimensions for variable in NetCDF file logical :: do_subset !Will variable subsetting be done? Answer provided by !var_subset_check subroutine. @@ -1889,7 +1887,7 @@ subroutine get_netcdf_var_char_2d(this, varname, var, errmsg, errcode, start, co !Check if variable subsetting is requested and valid: !Note that this involves ignoring the first dimension, !which is just the character length: - call var_subset_check(varname, var_ndims, dim_sizes(2:), do_subset, alloc_dims, errmsg, errcode, start, count) + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1900,7 +1898,7 @@ subroutine get_netcdf_var_char_2d(this, varname, var, errmsg, errcode, start, co !read-in the NetCDF data. Note that the first dimension !of dim_sizes is the length of the character variable, !and is not included in the 'alloc_dims' array. - allocate(character(dim_sizes(1)) :: var(alloc_dims(1), alloc_dims(2)), stat=errcode, errmsg=errmsg) + allocate(character(alloc_dims(1)) :: var(alloc_dims(2), alloc_dims(3)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -1971,14 +1969,12 @@ subroutine get_netcdf_var_char_3d(this, varname, var, errmsg, errcode, start, co !Ideally the actual type would be checked and handled !differently, but for now just confirm it's a character !array and check for ndims = rank+1 - integer, parameter :: var_ndims = 3 !Number of expected dimensions for variable in NetCDF file + integer, parameter :: var_ndims = 4 !Number of expected dimensions for variable in NetCDF file logical :: do_subset !Will variable subsetting be done? Answer provided by !var_subset_check subroutine. !---------------------- - - !Check if file is open: if(.not.this%sima_pio_fh%is_file_open) then !File isn't actually open, so throw an error @@ -2037,7 +2033,7 @@ subroutine get_netcdf_var_char_3d(this, varname, var, errmsg, errcode, start, co !Check if variable subsetting is requested and valid: !Note that this involves ignoring the first dimension, !which is just the character length: - call var_subset_check(varname, var_ndims, dim_sizes(2:), do_subset, alloc_dims, errmsg, errcode, start, count) + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -2048,7 +2044,7 @@ subroutine get_netcdf_var_char_3d(this, varname, var, errmsg, errcode, start, co !read-in the NetCDF data. Note that the first dimension !of dim_sizes is the length of the character variable, !and is not included in the 'alloc_dims' array. - allocate(character(dim_sizes(1)) :: var(alloc_dims(1), alloc_dims(2), alloc_dims(3)), stat=errcode, errmsg=errmsg) + allocate(character(alloc_dims(1)) :: var(alloc_dims(2), alloc_dims(3), alloc_dims(4)), stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -2119,7 +2115,7 @@ subroutine get_netcdf_var_char_4d(this, varname, var, errmsg, errcode, start, co !Ideally the actual type would be checked and handled !differently, but for now just confirm it's a character !array and check for ndims = rank+1 - integer, parameter :: var_ndims = 4 !Number of expected dimensions for variable in NetCDF file + integer, parameter :: var_ndims = 5 !Number of expected dimensions for variable in NetCDF file logical :: do_subset !Will variable subsetting be done? Answer provided by !var_subset_check subroutine. @@ -2183,7 +2179,7 @@ subroutine get_netcdf_var_char_4d(this, varname, var, errmsg, errcode, start, co !Check if variable subsetting is requested and valid !Note that this involves ignoring the first dimension, !which is just the character length: - call var_subset_check(varname, var_ndims, dim_sizes(2:), do_subset, alloc_dims, errmsg, errcode, start, count) + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -2194,7 +2190,7 @@ subroutine get_netcdf_var_char_4d(this, varname, var, errmsg, errcode, start, co !read-in the NetCDF data. Note that the first dimension !of dim_sizes is the length of the character variable, !and is not included in the 'alloc_dims' array. - allocate(character(dim_sizes(1)) :: var(alloc_dims(1), alloc_dims(2), alloc_dims(3), alloc_dims(4)), & + allocate(character(alloc_dims(1)) :: var(alloc_dims(2), alloc_dims(3), alloc_dims(4), alloc_dims(5)), & stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: @@ -2266,7 +2262,7 @@ subroutine get_netcdf_var_char_5d(this, varname, var, errmsg, errcode, start, co !Ideally the actual type would be checked and handled !differently, but for now just confirm it's a character !array and check for ndims = rank+1 - integer, parameter :: var_ndims = 5 !Number of expected dimensions for variable in NetCDF file + integer, parameter :: var_ndims = 6 !Number of expected dimensions for variable in NetCDF file logical :: do_subset !Will variable subsetting be done? Answer provided by !var_subset_check subroutine. @@ -2330,7 +2326,7 @@ subroutine get_netcdf_var_char_5d(this, varname, var, errmsg, errcode, start, co !Check if variable subsetting is requested and valid !Note that this involves ignoring the first dimension, !which is just the character length: - call var_subset_check(varname, var_ndims, dim_sizes(2:), do_subset, alloc_dims, errmsg, errcode, start, count) + call var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims, errmsg, errcode, start, count) if (errcode /= 0) then !Reset PIO back to original error handling method: call pio_seterrorhandling(pio_file_handle, err_handling) @@ -2341,7 +2337,7 @@ subroutine get_netcdf_var_char_5d(this, varname, var, errmsg, errcode, start, co !read-in the NetCDF data. Note that the first dimension !of dim_sizes is the length of the character variable, !and is not included in the 'alloc_dims' array. - allocate(character(dim_sizes(1)) :: var(alloc_dims(1), alloc_dims(2), alloc_dims(3), alloc_dims(4), alloc_dims(5)), & + allocate(character(alloc_dims(1)) :: var(alloc_dims(2), alloc_dims(3), alloc_dims(4), alloc_dims(5), alloc_dims(6)), & stat=errcode, errmsg=errmsg) if(errcode /= 0) then !Reset PIO back to original error handling method: diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 78360fed..68c9e236 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1562,12 +1562,12 @@ subroutine test_pio_reader_1d_char_full_subset_read() end do ! Check that the variable's values are correct: - @assertEqual("cfc11", trim(gas_names(10))) - @assertEqual("cfc12", trim(gas_names(11))) - @assertEqual("cfc22", trim(gas_names(12))) - @assertEqual("hfc143a", trim(gas_names(13))) - @assertEqual("hfc125", trim(gas_names(14))) - @assertEqual("hfc23", trim(gas_names(15))) + @assertEqual("cfc11", trim(gas_names(1))) + @assertEqual("cfc12", trim(gas_names(2))) + @assertEqual("cfc22", trim(gas_names(3))) + @assertEqual("hfc143a", trim(gas_names(4))) + @assertEqual("hfc125", trim(gas_names(5))) + @assertEqual("hfc23", trim(gas_names(6))) ! Perform test cleanup: From ab49b90508fad20b0f9d65c5ce27a81b37959fb1 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 22 Aug 2025 14:36:34 -0600 Subject: [PATCH 065/150] Fix error message in unit test. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 68c9e236..39621c93 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1173,7 +1173,7 @@ subroutine test_pio_reader_char_get_var_bad_rank() ! Expected error message: character(len=*), parameter :: expected_err_msg = & - "Variable 'gas_names' isn't declared with the correct number of dimensions. Expected 1 dimensions, but is declared with 2 dimensions." + "Variable 'gas_names' isn't declared with the correct number of dimensions. Expected 2 dimensions, but is declared with 3 dimensions." ! Begin test: From 0e5fb364e11556f9b40dde3cb3e87006e200bc4b Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 25 Aug 2025 10:24:44 -0600 Subject: [PATCH 066/150] Remove fork/branch from Github Actions workflow. --- .github/workflows/fortran_unit_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/fortran_unit_tests.yml b/.github/workflows/fortran_unit_tests.yml index 064c7cce..c2143ac7 100644 --- a/.github/workflows/fortran_unit_tests.yml +++ b/.github/workflows/fortran_unit_tests.yml @@ -5,7 +5,6 @@ on: branches: - development - main - - pio_reader_subset pull_request: workflow_dispatch: From f65ec49fb1b6f9c36003b48aeae1891dab32c667 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 25 Aug 2025 14:15:57 -0600 Subject: [PATCH 067/150] Remove leftover debugging code. --- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index 39621c93..49e5f68d 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1532,8 +1532,6 @@ subroutine test_pio_reader_1d_char_full_subset_read() character(len=*), parameter :: fname = & "../../../rrtmgp-data/rrtmgp-gas-sw-g112.nc" - integer :: i !DEBUGGING -JN - ! Begin test: reader => create_netcdf_reader_t() @@ -1556,11 +1554,6 @@ subroutine test_pio_reader_1d_char_full_subset_read() @assertEqual(32, len(gas_names)) @assertEqual(6, size(gas_names)) - ! DEBUGGING -JN - do i=1, size(gas_names) - write(*,*) "gas_names(", i, ") = '", trim(gas_names(i)), "'" - end do - ! Check that the variable's values are correct: @assertEqual("cfc11", trim(gas_names(1))) @assertEqual("cfc12", trim(gas_names(2))) @@ -1569,7 +1562,6 @@ subroutine test_pio_reader_1d_char_full_subset_read() @assertEqual("hfc125", trim(gas_names(5))) @assertEqual("hfc23", trim(gas_names(6))) - ! Perform test cleanup: deallocate(gas_names) call reader%close_file(errmsg, errcode) @@ -1680,7 +1672,7 @@ subroutine test_pio_reader_missing_count_argument() end subroutine test_pio_reader_missing_count_argument -!----------------------------------------- +!----------------------------------------- @test subroutine test_pio_reader_0d_real_subset_err() From 3e6eeaa680e4e64ee1ee3b8c573922caa585ae24 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Tue, 26 Aug 2025 17:27:59 -0600 Subject: [PATCH 068/150] update to dev tag --- .gitmodules | 2 +- src/physics/ncar_ccpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 181a4961..de395262 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = 28d69e2cc4d666611f1adc0dafd66ad112fc850d + fxtag = 493db339ac9acfd73125d48e96a6cf8f138a0589 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 28d69e2c..493db339 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 28d69e2cc4d666611f1adc0dafd66ad112fc850d +Subproject commit 493db339ac9acfd73125d48e96a6cf8f138a0589 From 2cfe2971b05dcbe81fca4b1157a5ca8c408e7f81 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Tue, 26 Aug 2025 23:59:01 -0400 Subject: [PATCH 069/150] Update registry for gw_drag (orographic at least, CAM4) --- src/data/registry.xml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/data/registry.xml b/src/data/registry.xml index 0371676a..cd3850b8 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -19,6 +19,8 @@ $SRCROOT/src/data/ref_pres.meta $SRCROOT/src/dynamics/utils/vert_coord.meta $SRCROOT/src/dynamics/utils/hycoef.meta + + $SRCROOT/src/physics/ncar_ccpp/to_be_ccppized/coords_1d.meta @@ -1313,6 +1315,7 @@ units="None" type="cam_out_t"> + pbuf_LCWAT + + + vertical_layer_dimension + 1.0_kind_phys + + + horizontal_dimension vertical_interface_dimension + 0.0_kind_phys + pbuf_kvt + + + horizontal_dimension + 0.0_kind_phys + pbuf_SGH + + Date: Fri, 29 Aug 2025 15:03:10 -0400 Subject: [PATCH 070/150] Add host side support module for reading gravity wave ridge data. --- cime_config/namelist_definition_cam.xml | 13 + src/control/cam_comp.F90 | 4 + src/control/runtime_opts.F90 | 3 +- src/data/registry.xml | 9 +- src/dynamics/se/dyn_comp.F90 | 4 +- .../utils/gravity_wave_drag_ridge_read.F90 | 339 ++++++++++++++++++ .../utils/gravity_wave_drag_ridge_read.meta | 82 +++++ src/physics/utils/tropopause_climo_read.F90 | 1 + 8 files changed, 444 insertions(+), 11 deletions(-) create mode 100644 src/physics/utils/gravity_wave_drag_ridge_read.F90 create mode 100644 src/physics/utils/gravity_wave_drag_ridge_read.meta diff --git a/cime_config/namelist_definition_cam.xml b/cime_config/namelist_definition_cam.xml index 52d8d331..7379936b 100644 --- a/cime_config/namelist_definition_cam.xml +++ b/cime_config/namelist_definition_cam.xml @@ -377,6 +377,19 @@ + + + char*256 + gravity_wave_drag + gw_drag_input_nl + + Full pathname of boundary dataset for meso-gamma ridges. + + + UNSET_PATH + + + diff --git a/src/control/cam_comp.F90 b/src/control/cam_comp.F90 index 3af07091..4c23eb52 100644 --- a/src/control/cam_comp.F90 +++ b/src/control/cam_comp.F90 @@ -103,6 +103,7 @@ subroutine cam_init(caseid, ctitle, model_doi_url, & use vert_coord, only: pver use phys_vars_init_check, only: mark_as_initialized use tropopause_climo_read, only: tropopause_climo_read_file + use gravity_wave_drag_ridge_read, only: gravity_wave_drag_ridge_read_file use orbital_data, only: orbital_data_init use ccpp_kinds, only: kind_phys use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t @@ -248,6 +249,9 @@ subroutine cam_init(caseid, ctitle, model_doi_url, & ! Read tropopause climatology call tropopause_climo_read_file() + ! Read gravity wave drag data for ridge parameterization + call gravity_wave_drag_ridge_read_file() + ! TEMPORARY: Prescribe realistic but inaccurate physical quantities ! necessary for MUSICA that are currently unavailable in CAM-SIMA. ! diff --git a/src/control/runtime_opts.F90 b/src/control/runtime_opts.F90 index 7eb79f32..0951ca1f 100644 --- a/src/control/runtime_opts.F90 +++ b/src/control/runtime_opts.F90 @@ -45,6 +45,7 @@ subroutine read_namelist(nlfilename, single_column, scmlat, scmlon) use inic_analytic_utils, only: analytic_ic_readnl use tropopause_climo_read, only: tropopause_climo_readnl + use gravity_wave_drag_ridge_read, only: gravity_wave_drag_ridge_read_readnl ! use tracers, only: tracers_readnl ! use nudging, only: nudging_readnl @@ -104,7 +105,7 @@ subroutine read_namelist(nlfilename, single_column, scmlat, scmlon) call tropopause_climo_readnl(nlfilename) ! call scam_readnl(nlfilename, single_column, scmlat, scmlon) ! call nudging_readnl(nlfilename) - + call gravity_wave_drag_ridge_read_readnl(nlfilename) call dyn_readnl(nlfilename) ! Read the namelists for active physics schemes diff --git a/src/data/registry.xml b/src/data/registry.xml index cd3850b8..5decdf8f 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -13,6 +13,7 @@ $SRCROOT/src/physics/utils/physics_grid.meta $SRCROOT/src/physics/utils/cam_constituents.meta $SRCROOT/src/physics/utils/tropopause_climo_read.meta + $SRCROOT/src/physics/utils/gravity_wave_drag_ridge_read.meta $SRCROOT/src/data/air_composition.meta $SRCROOT/src/data/cam_thermo.meta $SRCROOT/src/data/cam_thermo_formula.meta @@ -1392,14 +1393,6 @@ - - vertical_layer_dimension - 1.0_kind_phys - \section arg_table_gravity_wave_drag_ridge_read Argument Table +!! \htmlinclude gravity_wave_drag_ridge_read.html + integer, parameter, public :: prdg = 16 ! # of ridges + + ! Meso-Beta ridges: + real(kind_phys), allocatable, public :: rdg_gbxar (:) ! Grid box area (ncol) [km2] + real(kind_phys), allocatable, public :: rdg_isovar(:) ! (B) Isotropic variance (ncol) [m] + real(kind_phys), allocatable, public :: rdg_isowgt(:) ! (B) Isotropic weight (ncol) [1] + real(kind_phys), allocatable, public :: rdg_hwdth (:, :) ! (B) Ridge Half-widths (ncol,prdg) [km] + real(kind_phys), allocatable, public :: rdg_clngt (:, :) ! (B) Ridge length (ncol,prdg) [km] + real(kind_phys), allocatable, public :: rdg_mxdis (:, :) ! (B) Ridge/obstacle height (ncol,prdg) [m] + real(kind_phys), allocatable, public :: rdg_anixy (:, :) ! (B) Ridge anisotropy (ncol,prdg) [1] + real(kind_phys), allocatable, public :: rdg_angll (:, :) ! (B) Ridge clockwise angle w.r.t. N-S direction (ncol,prdg) [degrees] + + ! Meso-Gamma ridges: + real(kind_phys), allocatable, public :: rdg_gbxarg(:) ! Grid box area (ncol) [km2] + real(kind_phys), allocatable, public :: rdg_hwdthg(:, :) ! (G) Ridge Half-widths (ncol,prdg) [km] + real(kind_phys), allocatable, public :: rdg_clngtg(:, :) ! (G) Ridge length (ncol,prdg) [km] + real(kind_phys), allocatable, public :: rdg_mxdisg(:, :) ! (G) Ridge/obstacle height (ncol,prdg) [m] + real(kind_phys), allocatable, public :: rdg_anixyg(:, :) ! (G) Ridge anisotropy (ncol,prdg) [1] + real(kind_phys), allocatable, public :: rdg_angllg(:, :) ! (G) Ridge clockwise angle w.r.t. N-S direction (ncol,prdg) [degrees] + +contains + + subroutine gravity_wave_drag_ridge_read_readnl(nlfile) + use shr_nl_mod, only: find_group_name => shr_nl_find_group_name + use shr_kind_mod, only: shr_kind_cm + use mpi, only: mpi_character + use spmd_utils, only: mpicom + use cam_logfile, only: iulog + use cam_abortutils, only: endrun + use spmd_utils, only: masterproc + use cam_initfiles, only: unset_path_str + + ! topography dataset is available in this module: + use cam_initfiles, only: cam_bnd_topo => bnd_topo + ! and does not need to be re-read from namelist. + + ! filepath for file containing namelist input + character(len=*), intent(in) :: nlfile + + ! Local variables + integer :: unitn, errflg + character(len=*), parameter :: subname = 'gravity_wave_drag_ridge_read_readnl' + character(len=shr_kind_cm) :: errmsg + + namelist /gw_drag_input_nl/ bnd_rdggm + + errmsg = '' + errflg = 0 + + if (masterproc) then + open(newunit=unitn, file=trim(nlfile), status='old') + call find_group_name(unitn, 'gw_drag_input_nl', status=errflg) + if (errflg == 0) then + read(unitn, gw_drag_input_nl, iostat=errflg, iomsg=errmsg) + if (errflg /= 0) then + call endrun(subname // ':: ERROR reading namelist:' // errmsg) + end if + end if + close(unitn) + end if + + ! Broadcast namelist variables + call mpi_bcast(bnd_rdggm, len(bnd_rdggm), mpi_character, 0, mpicom, errflg) + + ! Retrieve topo file location: + bnd_topo = cam_bnd_topo + + ! Print out namelist variables + if (masterproc) then + write(iulog,*) subname, ' options:' + if(bnd_topo /= unset_path_str) then + write(iulog,*) ' Gravity wave meso-Beta ridge topo file: ', trim(bnd_topo) + else + write(iulog,*) ' Gravity wave meso-Beta ridge input data unavailable.' + endif + + if(bnd_rdggm /= unset_path_str) then + write(iulog,*) ' Gravity wave meso-Gamma ridge input file: ', trim(bnd_rdggm) + else + write(iulog,*) ' Gravity wave meso-Gamma ridge input data unavailable.' + endif + endif + end subroutine gravity_wave_drag_ridge_read_readnl + + subroutine gravity_wave_drag_ridge_read_file() + use spmd_utils, only: masterproc + use cam_logfile, only: iulog + use cam_abortutils, only: endrun, check_allocate + use pio, only: file_desc_t, pio_nowrite + use cam_pio_utils, only: cam_pio_openfile, cam_pio_closefile + use ioFileMod, only: cam_get_file + use cam_initfiles, only: topo_file_get_id + use cam_field_read, only: cam_read_field + use physconst, only: rearth + use cam_initfiles, only: unset_path_str + use physics_grid, only: ncol => columns_on_task + + ! Local variables + type(file_desc_t), pointer :: fh_topo + type(file_desc_t), pointer :: fh_rdggm + integer :: errflg + character(len=512) :: errmsg + character(len=*), parameter :: subname = 'gravity_wave_drag_ridge_read_file' + + logical :: found + logical :: has_gbxar_from_topo + + errmsg = '' + errflg = 0 + + has_gbxar_from_topo = .false. + + ! Do we have meso-Beta file? + if(bnd_topo /= unset_path_str) then + call cam_get_file(bnd_topo, bnd_topo_loc) + + ! Try getting from initfiles first. + fh_topo => topo_file_get_id() + if(.not. associated(fh_topo)) then + ! I think this case will never be hit in SIMA. + ! There was a fallback in CAM. + call endrun(trim(subname) // ": fh_topo from cam_initfiles is not available and this is not implemented.") + endif + + if(masterproc) then + write (iulog,*) trim(subname)//': Reading meso-Beta ridge data from ', trim(bnd_topo_loc) + endif + + ! Allocate and initialize data to zeros. + allocate(rdg_gbxar(ncol), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_gbxar', errmsg=errmsg) + allocate(rdg_isovar(ncol), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_isovar', errmsg=errmsg) + allocate(rdg_isowgt(ncol), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_isowgt', errmsg=errmsg) + allocate(rdg_hwdth(ncol, prdg), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_hwdth', errmsg=errmsg) + allocate(rdg_clngt(ncol, prdg), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_clngt', errmsg=errmsg) + allocate(rdg_mxdis(ncol, prdg), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_mxdis', errmsg=errmsg) + allocate(rdg_anixy(ncol, prdg), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_anixy', errmsg=errmsg) + allocate(rdg_angll(ncol, prdg), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_angll', errmsg=errmsg) + + rdg_gbxar(:) = 0._kind_phys + rdg_isovar(:) = 0._kind_phys + rdg_isowgt(:) = 0._kind_phys + rdg_hwdth(:,:) = 0._kind_phys + rdg_clngt(:,:) = 0._kind_phys + rdg_mxdis(:,:) = 0._kind_phys + rdg_anixy(:,:) = 0._kind_phys + rdg_angll(:,:) = 0._kind_phys + + ! Use cam_field_read to read the file data: + + ! Read required 1D field: GBXAR (grid box area) + call cam_read_field('GBXAR', fh_topo, rdg_gbxar, found) + if(.not. found) then + call endrun(trim(subname) // ': GBXAR not found in input file') + endif + ! Convert from m2 to km2 + rdg_gbxar = rdg_gbxar * (rearth/1000._kind_phys) * (rearth/1000._kind_phys) + has_gbxar_from_topo = .true. + + ! Read optional 1D field: ISOVAR (isotropic variance) + call cam_read_field('ISOVAR', fh_topo, rdg_isovar, found) + if(.not. found) then + if(masterproc) then + write(iulog,*) trim(subname) // ': ISOVAR not found in topo file, using zero values' + endif + ! rdg_isovar already initialized to zero above + endif + + ! Read optional 1D field: ISOWGT (isotropic weight) + call cam_read_field('ISOWGT', fh_topo, rdg_isowgt, found) + if(.not. found) then + if(masterproc) then + write(iulog,*) trim(subname) // ': ISOWGT not found in topo file, using zero values' + endif + ! rdg_isowgt already initialized to zero above + endif + + ! Read required 2D field: HWDTH (ridge half-widths) + call cam_read_field('HWDTH', fh_topo, rdg_hwdth, found, dim3name='nrdg', dim3_bnds=(/1, prdg/)) + if(.not. found) then + call endrun(trim(subname) // ': HWDTH not found in input file') + endif + + ! Read required 2D field: CLNGT (ridge length) + call cam_read_field('CLNGT', fh_topo, rdg_clngt, found, dim3name='nrdg', dim3_bnds=(/1, prdg/)) + if(.not. found) then + call endrun(trim(subname) // ': CLNGT not found in input file') + endif + + ! Read required 2D field: MXDIS (ridge/obstacle height) + call cam_read_field('MXDIS', fh_topo, rdg_mxdis, found, dim3name='nrdg', dim3_bnds=(/1, prdg/)) + if(.not. found) then + call endrun(trim(subname) // ': MXDIS not found in input file') + endif + + ! Read required 2D field: ANIXY (ridge anisotropy) + call cam_read_field('ANIXY', fh_topo, rdg_anixy, found, dim3name='nrdg', dim3_bnds=(/1, prdg/)) + if(.not. found) then + call endrun(trim(subname) // ': ANIXY not found in input file') + endif + + ! Read required 2D field: ANGLL (ridge clockwise angle w.r.t. N-S direction) + call cam_read_field('ANGLL', fh_topo, rdg_angll, found, dim3name='nrdg', dim3_bnds=(/1, prdg/)) + if(.not. found) then + call endrun(trim(subname) // ': ANGLL not found in input file') + endif + endif + + ! Do we have meso-Gamma file? + if(bnd_rdggm /= unset_path_str) then + call cam_get_file(bnd_rdggm, bnd_rdggm_loc) + call cam_pio_openfile(fh_rdggm, bnd_rdggm_loc, pio_nowrite) + + if(masterproc) then + write (iulog,*) trim(subname)//': Reading meso-Gamma ridge data from ', trim(bnd_topo_loc) + endif + + ! Allocate meso-Gamma ridge data arrays + allocate(rdg_gbxarg(ncol), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_gbxarg', errmsg=errmsg) + allocate(rdg_hwdthg(ncol, prdg), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_hwdthg', errmsg=errmsg) + allocate(rdg_clngtg(ncol, prdg), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_clngtg', errmsg=errmsg) + allocate(rdg_mxdisg(ncol, prdg), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_mxdisg', errmsg=errmsg) + allocate(rdg_anixyg(ncol, prdg), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_anixyg', errmsg=errmsg) + allocate(rdg_angllg(ncol, prdg), stat=errflg, errmsg=errmsg) + call check_allocate(errflg, subname, 'rdg_angllg', errmsg=errmsg) + + ! Initialize all gamma fields to zero first + rdg_gbxarg(:) = 0._kind_phys + rdg_hwdthg(:,:) = 0._kind_phys + rdg_clngtg(:,:) = 0._kind_phys + rdg_mxdisg(:,:) = 0._kind_phys + rdg_anixyg(:,:) = 0._kind_phys + rdg_angllg(:,:) = 0._kind_phys + + if(.not. has_gbxar_from_topo) then + call cam_read_field('GBXAR', fh_rdggm, rdg_gbxarg, found) + if(.not. found) then + call endrun(trim(subname) // ': GBXAR not found in neither topo or gamma ridge file') + endif + ! Convert from m2 to km2 + rdg_gbxarg = rdg_gbxarg * (rearth/1000._kind_phys) * (rearth/1000._kind_phys) + else + if(masterproc) then + write(iulog,*) trim(subname) // ': Using GBXAR from topo file, skipping gamma file GBXAR' + endif + endif + + ! ISOVAR and ISOWGT are intentionally not read from gamma file, + ! because (1) they are unavailable, and (2) the original code did not read them in + ! and left them as dangling pointers. See gw_drag_cam CAM interface, + ! gravity_wave_drag_ridge_gamma_run implementation. + + ! Read required 2D field: HWDTH (ridge half-widths gamma) + call cam_read_field('HWDTH', fh_rdggm, rdg_hwdthg, found, dim3name='nrdg', dim3_bnds=(/1, prdg/)) + if(.not. found) then + call endrun(trim(subname) // ': HWDTH not found in gamma ridge file') + endif + + ! Read required 2D field: CLNGT (ridge length gamma) + call cam_read_field('CLNGT', fh_rdggm, rdg_clngtg, found, dim3name='nrdg', dim3_bnds=(/1, prdg/)) + if(.not. found) then + call endrun(trim(subname) // ': CLNGT not found in gamma ridge file') + endif + + ! Read required 2D field: MXDIS (ridge/obstacle height gamma) + call cam_read_field('MXDIS', fh_rdggm, rdg_mxdisg, found, dim3name='nrdg', dim3_bnds=(/1, prdg/)) + if(.not. found) then + call endrun(trim(subname) // ': MXDIS not found in gamma ridge file') + endif + + ! Apply negative value correction for gamma ridge maximum displacement + where (rdg_mxdisg < 0._kind_phys) + rdg_mxdisg = 0._kind_phys + end where + + ! Read required 2D field: ANIXY (ridge anisotropy gamma) + call cam_read_field('ANIXY', fh_rdggm, rdg_anixyg, found, dim3name='nrdg', dim3_bnds=(/1, prdg/)) + if(.not. found) then + call endrun(trim(subname) // ': ANIXY not found in gamma ridge file') + endif + + ! Read required 2D field: ANGLL (ridge clockwise angle w.r.t. N-S direction gamma) + call cam_read_field('ANGLL', fh_rdggm, rdg_angllg, found, dim3name='nrdg', dim3_bnds=(/1, prdg/)) + if(.not. found) then + call endrun(trim(subname) // ': ANGLL not found in gamma ridge file') + endif + + call cam_pio_closefile(fh_rdggm) + deallocate(fh_rdggm) + + endif + end subroutine gravity_wave_drag_ridge_read_file + +end module gravity_wave_drag_ridge_read diff --git a/src/physics/utils/gravity_wave_drag_ridge_read.meta b/src/physics/utils/gravity_wave_drag_ridge_read.meta new file mode 100644 index 00000000..ab93dcf9 --- /dev/null +++ b/src/physics/utils/gravity_wave_drag_ridge_read.meta @@ -0,0 +1,82 @@ +[ccpp-table-properties] + name = gravity_wave_drag_ridge_read + type = module + +[ccpp-arg-table] + name = gravity_wave_drag_ridge_read + type = module +[ prdg ] + standard_name = number_of_ridges_in_ridge_gravity_wave_drag_tbd + units = count + type = integer + dimensions = () +[ rdg_gbxar ] + standard_name = grid_box_area_for_meso_beta_gravity_wave_drag_tbd + units = km2 + type = real | kind = kind_phys + dimensions = (horizontal_dimension) +[ rdg_isovar ] + standard_name = isotropic_variance_for_meso_beta_gravity_wave_drag_tbd + units = m + type = real | kind = kind_phys + dimensions = (horizontal_dimension) +[ rdg_isowgt ] + standard_name = isotropic_weight_for_meso_beta_gravity_wave_drag_tbd + units = 1 + type = real | kind = kind_phys + dimensions = (horizontal_dimension) +[ rdg_hwdth ] + standard_name = ridge_half_width_for_meso_beta_gravity_wave_drag_tbd + units = km + type = real | kind = kind_phys + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) +[ rdg_clngt ] + standard_name = ridge_length_for_meso_beta_gravity_wave_drag_tbd + units = km + type = real | kind = kind_phys + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) +[ rdg_mxdis ] + standard_name = ridge_obstacle_height_for_meso_beta_gravity_wave_drag_tbd + units = m + type = real | kind = kind_phys + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) +[ rdg_anixy ] + standard_name = ridge_anisotropy_for_meso_beta_gravity_wave_drag_tbd + units = 1 + type = real | kind = kind_phys + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) +[ rdg_angll ] + standard_name = ridge_clockwise_angle_from_north_for_meso_beta_gravity_wave_drag_tbd + units = degree + type = real | kind = kind_phys + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) +[ rdg_gbxarg ] + standard_name = grid_box_area_for_meso_gamma_gravity_wave_drag_tbd + units = km2 + type = real | kind = kind_phys + dimensions = (horizontal_dimension) +[ rdg_hwdthg ] + standard_name = ridge_half_width_for_meso_gamma_gravity_wave_drag_tbd + units = km + type = real | kind = kind_phys + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) +[ rdg_clngtg ] + standard_name = ridge_length_for_meso_gamma_gravity_wave_drag_tbd + units = km + type = real | kind = kind_phys + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) +[ rdg_mxdisg ] + standard_name = ridge_obstacle_height_for_meso_gamma_gravity_wave_drag_tbd + units = m + type = real | kind = kind_phys + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) +[ rdg_anixyg ] + standard_name = ridge_anisotropy_for_meso_gamma_gravity_wave_drag_tbd + units = 1 + type = real | kind = kind_phys + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) +[ rdg_angllg ] + standard_name = ridge_clockwise_angle_from_north_for_meso_gamma_gravity_wave_drag_tbd + units = degree + type = real | kind = kind_phys + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) diff --git a/src/physics/utils/tropopause_climo_read.F90 b/src/physics/utils/tropopause_climo_read.F90 index f423b9a7..87af3d08 100644 --- a/src/physics/utils/tropopause_climo_read.F90 +++ b/src/physics/utils/tropopause_climo_read.F90 @@ -13,6 +13,7 @@ module tropopause_climo_read implicit none private + save public :: tropopause_climo_readnl public :: tropopause_climo_read_file From be2b7cc9a6c63ac6a66c6dce4dfd2045a12b3ffb Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Fri, 29 Aug 2025 15:04:42 -0400 Subject: [PATCH 071/150] Revise comment at head of module --- src/physics/utils/gravity_wave_drag_ridge_read.F90 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/physics/utils/gravity_wave_drag_ridge_read.F90 b/src/physics/utils/gravity_wave_drag_ridge_read.F90 index 9ce06d84..3eb108bc 100644 --- a/src/physics/utils/gravity_wave_drag_ridge_read.F90 +++ b/src/physics/utils/gravity_wave_drag_ridge_read.F90 @@ -3,8 +3,10 @@ ! ! Remarks: this module is not CCPP-ized but is written specifically for ! grid decomposition-aware I/O in CAM-SIMA; it also has variables that -! are not a grid dimension (nrdg = 16), so it uses the PIO utilities -! directly. This module can be a useful reference for how to provide +! are not a grid dimension (prdg = 16) that are read by specifying the +! custom dimension. It also makes that dimension available to the CCPP +! framework (prdg). +! This module can be a useful reference for how to provide ! gridded data to underlying CCPP schemes via the CAM PIO decomposition. ! (hplin, 8/28/25) module gravity_wave_drag_ridge_read From 5b62e05eef162476d7a5e97dc866cc46273e20ba Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Wed, 10 Sep 2025 09:59:50 -0600 Subject: [PATCH 072/150] Update ridge input standard names --- .../utils/gravity_wave_drag_ridge_read.meta | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/physics/utils/gravity_wave_drag_ridge_read.meta b/src/physics/utils/gravity_wave_drag_ridge_read.meta index ab93dcf9..2d62af8c 100644 --- a/src/physics/utils/gravity_wave_drag_ridge_read.meta +++ b/src/physics/utils/gravity_wave_drag_ridge_read.meta @@ -11,72 +11,72 @@ type = integer dimensions = () [ rdg_gbxar ] - standard_name = grid_box_area_for_meso_beta_gravity_wave_drag_tbd + standard_name = grid_box_area_for_beta_ridge_gravity_wave_drag_tbd units = km2 type = real | kind = kind_phys dimensions = (horizontal_dimension) [ rdg_isovar ] - standard_name = isotropic_variance_for_meso_beta_gravity_wave_drag_tbd + standard_name = isotropic_variance_for_beta_ridge_gravity_wave_drag_tbd units = m type = real | kind = kind_phys dimensions = (horizontal_dimension) [ rdg_isowgt ] - standard_name = isotropic_weight_for_meso_beta_gravity_wave_drag_tbd + standard_name = isotropic_weight_for_beta_ridge_gravity_wave_drag_tbd units = 1 type = real | kind = kind_phys dimensions = (horizontal_dimension) [ rdg_hwdth ] - standard_name = ridge_half_width_for_meso_beta_gravity_wave_drag_tbd + standard_name = ridge_half_width_for_beta_ridge_gravity_wave_drag_tbd units = km type = real | kind = kind_phys dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) [ rdg_clngt ] - standard_name = ridge_length_for_meso_beta_gravity_wave_drag_tbd + standard_name = ridge_length_for_beta_ridge_gravity_wave_drag_tbd units = km type = real | kind = kind_phys dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) [ rdg_mxdis ] - standard_name = ridge_obstacle_height_for_meso_beta_gravity_wave_drag_tbd + standard_name = ridge_obstacle_height_for_beta_ridge_gravity_wave_drag_tbd units = m type = real | kind = kind_phys dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) [ rdg_anixy ] - standard_name = ridge_anisotropy_for_meso_beta_gravity_wave_drag_tbd + standard_name = ridge_anisotropy_for_beta_ridge_gravity_wave_drag_tbd units = 1 type = real | kind = kind_phys dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) [ rdg_angll ] - standard_name = ridge_clockwise_angle_from_north_for_meso_beta_gravity_wave_drag_tbd + standard_name = ridge_clockwise_angle_from_north_for_beta_ridge_gravity_wave_drag_tbd units = degree type = real | kind = kind_phys dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) [ rdg_gbxarg ] - standard_name = grid_box_area_for_meso_gamma_gravity_wave_drag_tbd + standard_name = grid_box_area_for_gamma_ridge_gravity_wave_drag_tbd units = km2 type = real | kind = kind_phys dimensions = (horizontal_dimension) [ rdg_hwdthg ] - standard_name = ridge_half_width_for_meso_gamma_gravity_wave_drag_tbd + standard_name = ridge_half_width_for_gamma_ridge_gravity_wave_drag_tbd units = km type = real | kind = kind_phys dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) [ rdg_clngtg ] - standard_name = ridge_length_for_meso_gamma_gravity_wave_drag_tbd + standard_name = ridge_length_for_gamma_ridge_gravity_wave_drag_tbd units = km type = real | kind = kind_phys dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) [ rdg_mxdisg ] - standard_name = ridge_obstacle_height_for_meso_gamma_gravity_wave_drag_tbd + standard_name = ridge_obstacle_height_for_gamma_ridge_gravity_wave_drag_tbd units = m type = real | kind = kind_phys dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) [ rdg_anixyg ] - standard_name = ridge_anisotropy_for_meso_gamma_gravity_wave_drag_tbd + standard_name = ridge_anisotropy_for_gamma_ridge_gravity_wave_drag_tbd units = 1 type = real | kind = kind_phys dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) [ rdg_angllg ] - standard_name = ridge_clockwise_angle_from_north_for_meso_gamma_gravity_wave_drag_tbd + standard_name = ridge_clockwise_angle_from_north_for_gamma_ridge_gravity_wave_drag_tbd units = degree type = real | kind = kind_phys dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) From cc60fb5b489e864260e8c6bcaa82a2d2c7ca9fc1 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Wed, 10 Sep 2025 20:58:51 -0600 Subject: [PATCH 073/150] Misc fix: 3-D input field read with dycore enabled; cam_in/cam_out from physics data. Add FADIAB test Update comment --- cime_config/testdefs/testlist_cam.xml | 10 +++++++++ src/control/cam_comp.F90 | 18 ++++++++-------- src/cpl/nuopc/atm_comp_nuopc.F90 | 3 +-- src/utils/cam_field_read.F90 | 31 +++++++++++++++++++++------ 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index ccec7022..97573ab1 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -121,6 +121,16 @@ + + + + + + + + + + diff --git a/src/control/cam_comp.F90 b/src/control/cam_comp.F90 index 3af07091..28b76119 100644 --- a/src/control/cam_comp.F90 +++ b/src/control/cam_comp.F90 @@ -145,8 +145,8 @@ subroutine cam_init(caseid, ctitle, model_doi_url, & integer, intent(in) :: ref_ymd ! Reference date (YYYYMMDD) integer, intent(in) :: ref_tod ! Reference time of day (sec) - type(cam_out_t), pointer :: cam_out ! Output from CAM to surface - type(cam_in_t), pointer :: cam_in ! Merged input state to CAM + type(cam_out_t) :: cam_out ! Output from CAM to surface + type(cam_in_t) :: cam_in ! Merged input state to CAM ! Local variables character(len=cs) :: filein ! Input namelist filename @@ -357,8 +357,8 @@ subroutine cam_run1(cam_in, cam_out) use phys_comp, only: phys_run1 ! use ionosphere_interface, only: ionosphere_run1 - type(cam_in_t), pointer, intent(inout) :: cam_in ! Input from surface to CAM - type(cam_out_t), pointer, intent(inout) :: cam_out ! Output from CAM to surface + type(cam_in_t), intent(inout) :: cam_in ! Input from surface to CAM + type(cam_out_t), intent(inout) :: cam_out ! Output from CAM to surface !---------------------------------------------------------- ! first phase of ionosphere -- write to IC file if needed @@ -396,8 +396,8 @@ subroutine cam_run2(cam_out, cam_in) use stepon, only: stepon_run2 ! use ionosphere_interface, only: ionosphere_run2 - type(cam_out_t), pointer, intent(inout) :: cam_out ! Output from CAM to surface - type(cam_in_t), pointer, intent(inout) :: cam_in ! Input from surface to CAM + type(cam_out_t), intent(inout) :: cam_out ! Output from CAM to surface + type(cam_in_t), intent(inout) :: cam_in ! Input from surface to CAM ! ! Second phase of physics (after surface model update) @@ -439,7 +439,7 @@ subroutine cam_run3(cam_out) !----------------------------------------------------------------------- use stepon, only: stepon_run3 - type(cam_out_t), pointer, intent(inout) :: cam_out ! Output from CAM to surface + type(cam_out_t), intent(inout) :: cam_out ! Output from CAM to surface !----------------------------------------------------------------------- ! @@ -564,8 +564,8 @@ subroutine cam_final(cam_out, cam_in) ! ! Arguments ! - type(cam_out_t), pointer :: cam_out ! Output from CAM to surface - type(cam_in_t), pointer :: cam_in ! Input from merged surface to CAM + type(cam_out_t) :: cam_out ! Output from CAM to surface + type(cam_in_t) :: cam_in ! Input from merged surface to CAM !----------------------------------------------------------------------- diff --git a/src/cpl/nuopc/atm_comp_nuopc.F90 b/src/cpl/nuopc/atm_comp_nuopc.F90 index 4e7498b9..aa420652 100644 --- a/src/cpl/nuopc/atm_comp_nuopc.F90 +++ b/src/cpl/nuopc/atm_comp_nuopc.F90 @@ -44,6 +44,7 @@ module atm_comp_nuopc use cam_comp , only : cam_init, cam_run1, cam_run2, cam_run3, cam_run4, cam_final use cam_comp , only : cam_timestep_init, cam_timestep_final use physics_types , only : cam_out_t, cam_in_t + use physics_types , only : cam_out, cam_in ! use radiation , only : nextsw_cday !uncomment once radiation has been CCPP-ized -JN use cam_logfile , only : cam_set_log_unit, iulog use cam_abortutils , only : check_allocate @@ -108,8 +109,6 @@ module atm_comp_nuopc integer :: nthrds integer :: ierr ! allocate status integer , parameter :: dbug_flag = 0 - type(cam_in_t) , pointer :: cam_in - type(cam_out_t) , pointer :: cam_out integer , pointer :: dof(:) ! global index space decomposition character(len=256) :: rsfilename_spec_cam ! Filename specifier for restart surface file character(*) ,parameter :: modName = "(atm_comp_nuopc)" diff --git a/src/utils/cam_field_read.F90 b/src/utils/cam_field_read.F90 index af3e5080..12df2d54 100644 --- a/src/utils/cam_field_read.F90 +++ b/src/utils/cam_field_read.F90 @@ -788,12 +788,28 @@ subroutine infld_real8_3d(varname, ncid, field, readvar, dim3name, & ! ! Ensure that is configured correctly if (dim_bounds(2,2) < dim_bounds(2,1)) then - if (present(gridname)) then - write(errormsg, *) ': grid, ', trim(gridname), & - ' invalid for3D field' + if(unstruct) then + ! Unstructured grid expects that a second dimension does not + ! exist horizontally. Receive vertical dimension from caller + if (present(dim3_pos)) then + if ((dim3_pos < 1) .or. (dim3_pos > 3)) then + call safe_endrun(subname//': Bad value for dim3_pos') + end if + index = dim3_pos + else + index = 2 + end if + dim_bounds(index,1) = dim3_bnds(1) + dim_bounds(index,2) = dim3_bnds(2) else - write(errormsg, *) ': grid, physgrid, invalid for3D field' - end if + if (present(gridname)) then + write(errormsg, *) ': grid, ', trim(gridname), & + ' invalid for 3D field' + else + write(errormsg, *) ': grid, physgrid, invalid for 3D field' + end if + call safe_endrun(subname//errormsg) + endif else if (present(dim3_pos)) then if ((dim3_pos < 1) .or. (dim3_pos > 3)) then @@ -866,7 +882,10 @@ subroutine infld_real8_3d(varname, ncid, field, readvar, dim3name, & (grid_dimlens(1) * grid_dimlens(2)) call safe_endrun(subname//trim(errormsg)) end if - else + else if (unstruct) then + ! Initialize index offset to remove vertical dimension from dimlens + ! to be checked. + index = 0 do jndex = 1, target_ndims if (trim(file_dnames(jndex)) == trim(dim3name)) then ! The vertical dimension may be in between array dims From db335b5d8d4458000918b7f44f4d36a385a0f702 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Fri, 12 Sep 2025 12:48:37 -0400 Subject: [PATCH 074/150] Update cime_config/testdefs/testlist_cam.xml Co-authored-by: Jesse Nusbaumer --- cime_config/testdefs/testlist_cam.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index 97573ab1..2bbf396f 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -128,7 +128,7 @@ - + From 573cf9a57e4b5b2683c5f766c2de5e9e1f45769f Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Fri, 12 Sep 2025 10:48:56 -0600 Subject: [PATCH 075/150] Address review comments --- src/control/cam_comp.F90 | 4 ++-- src/cpl/nuopc/atm_comp_nuopc.F90 | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/control/cam_comp.F90 b/src/control/cam_comp.F90 index 28b76119..5947e932 100644 --- a/src/control/cam_comp.F90 +++ b/src/control/cam_comp.F90 @@ -145,8 +145,8 @@ subroutine cam_init(caseid, ctitle, model_doi_url, & integer, intent(in) :: ref_ymd ! Reference date (YYYYMMDD) integer, intent(in) :: ref_tod ! Reference time of day (sec) - type(cam_out_t) :: cam_out ! Output from CAM to surface - type(cam_in_t) :: cam_in ! Merged input state to CAM + type(cam_out_t), intent(inout) :: cam_out ! Output from CAM to surface + type(cam_in_t) , intent(inout) :: cam_in ! Merged input state to CAM ! Local variables character(len=cs) :: filein ! Input namelist filename diff --git a/src/cpl/nuopc/atm_comp_nuopc.F90 b/src/cpl/nuopc/atm_comp_nuopc.F90 index aa420652..bc202466 100644 --- a/src/cpl/nuopc/atm_comp_nuopc.F90 +++ b/src/cpl/nuopc/atm_comp_nuopc.F90 @@ -43,7 +43,6 @@ module atm_comp_nuopc use cam_instance , only : cam_instance_init, inst_suffix, inst_index use cam_comp , only : cam_init, cam_run1, cam_run2, cam_run3, cam_run4, cam_final use cam_comp , only : cam_timestep_init, cam_timestep_final - use physics_types , only : cam_out_t, cam_in_t use physics_types , only : cam_out, cam_in ! use radiation , only : nextsw_cday !uncomment once radiation has been CCPP-ized -JN use cam_logfile , only : cam_set_log_unit, iulog From d347cf38324022599965f1ff689517c1044b8f94 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Fri, 12 Sep 2025 15:36:29 -0600 Subject: [PATCH 076/150] Updated to remove unused cam_in/cam_outs --- src/control/cam_comp.F90 | 32 +++++++------------------------- src/cpl/nuopc/atm_comp_nuopc.F90 | 20 +++++++++----------- 2 files changed, 16 insertions(+), 36 deletions(-) diff --git a/src/control/cam_comp.F90 b/src/control/cam_comp.F90 index 5947e932..af139aba 100644 --- a/src/control/cam_comp.F90 +++ b/src/control/cam_comp.F90 @@ -75,8 +75,7 @@ subroutine cam_init(caseid, ctitle, model_doi_url, & eccen, obliqr, lambm0, mvelpp, & perpetual_run, perpetual_ymd, & dtime, start_ymd, start_tod, ref_ymd, ref_tod, & - stop_ymd, stop_tod, curr_ymd, curr_tod, & - cam_out, cam_in) + stop_ymd, stop_tod, curr_ymd, curr_tod) !----------------------------------------------------------------------- ! @@ -145,9 +144,6 @@ subroutine cam_init(caseid, ctitle, model_doi_url, & integer, intent(in) :: ref_ymd ! Reference date (YYYYMMDD) integer, intent(in) :: ref_tod ! Reference time of day (sec) - type(cam_out_t), intent(inout) :: cam_out ! Output from CAM to surface - type(cam_in_t) , intent(inout) :: cam_in ! Merged input state to CAM - ! Local variables character(len=cs) :: filein ! Input namelist filename integer :: errflg @@ -345,7 +341,7 @@ end subroutine cam_timestep_init ! !----------------------------------------------------------------------- ! - subroutine cam_run1(cam_in, cam_out) + subroutine cam_run1() !----------------------------------------------------------------------- ! ! Purpose: First phase of atmosphere model run method. @@ -357,9 +353,6 @@ subroutine cam_run1(cam_in, cam_out) use phys_comp, only: phys_run1 ! use ionosphere_interface, only: ionosphere_run1 - type(cam_in_t), intent(inout) :: cam_in ! Input from surface to CAM - type(cam_out_t), intent(inout) :: cam_out ! Output from CAM to surface - !---------------------------------------------------------- ! first phase of ionosphere -- write to IC file if needed !---------------------------------------------------------- @@ -381,7 +374,7 @@ end subroutine cam_run1 !----------------------------------------------------------------------- ! - subroutine cam_run2(cam_out, cam_in) + subroutine cam_run2() !----------------------------------------------------------------------- ! ! Purpose: Second phase of atmosphere model run method. @@ -396,9 +389,6 @@ subroutine cam_run2(cam_out, cam_in) use stepon, only: stepon_run2 ! use ionosphere_interface, only: ionosphere_run2 - type(cam_out_t), intent(inout) :: cam_out ! Output from CAM to surface - type(cam_in_t), intent(inout) :: cam_in ! Input from surface to CAM - ! ! Second phase of physics (after surface model update) ! @@ -428,7 +418,7 @@ end subroutine cam_run2 !----------------------------------------------------------------------- ! - subroutine cam_run3(cam_out) + subroutine cam_run3() !----------------------------------------------------------------------- ! ! Purpose: Third phase of atmosphere model run method. This consists @@ -438,8 +428,8 @@ subroutine cam_run3(cam_out) ! !----------------------------------------------------------------------- use stepon, only: stepon_run3 + use physics_types, only: cam_out ! Output from CAM to surface - type(cam_out_t), intent(inout) :: cam_out ! Output from CAM to surface !----------------------------------------------------------------------- ! @@ -457,7 +447,7 @@ end subroutine cam_run3 !----------------------------------------------------------------------- ! - subroutine cam_run4(cam_out, cam_in, rstwr, nlend, & + subroutine cam_run4(rstwr, nlend, & yr_spec, mon_spec, day_spec, sec_spec) !----------------------------------------------------------------------- @@ -470,8 +460,6 @@ subroutine cam_run4(cam_out, cam_in, rstwr, nlend, & ! use cam_restart, only: cam_write_restart ! use qneg_module, only: qneg_print_summary - type(cam_out_t), intent(inout) :: cam_out ! Output from CAM to surface - type(cam_in_t), intent(inout) :: cam_in ! Input from surface to CAM logical, intent(in) :: rstwr ! write restart file logical, intent(in) :: nlend ! this is final timestep integer, intent(in), optional :: yr_spec ! Simulation year @@ -549,7 +537,7 @@ end subroutine cam_timestep_final !----------------------------------------------------------------------- ! - subroutine cam_final(cam_out, cam_in) + subroutine cam_final() !----------------------------------------------------------------------- ! ! Purpose: CAM finalization. @@ -561,12 +549,6 @@ subroutine cam_final(cam_out, cam_in) ! use ionosphere_interface, only: ionosphere_final use cam_control_mod, only: initial_run - ! - ! Arguments - ! - type(cam_out_t) :: cam_out ! Output from CAM to surface - type(cam_in_t) :: cam_in ! Input from merged surface to CAM - !----------------------------------------------------------------------- call phys_final() diff --git a/src/cpl/nuopc/atm_comp_nuopc.F90 b/src/cpl/nuopc/atm_comp_nuopc.F90 index bc202466..b13cbe94 100644 --- a/src/cpl/nuopc/atm_comp_nuopc.F90 +++ b/src/cpl/nuopc/atm_comp_nuopc.F90 @@ -662,9 +662,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) stop_ymd=stop_ymd, & stop_tod=stop_tod, & curr_ymd=curr_ymd, & - curr_tod=curr_tod, & - cam_out=cam_out, & - cam_in=cam_in) + curr_tod=curr_tod) if (mediator_present) then @@ -916,7 +914,7 @@ subroutine DataInitialize(gcomp, rc) call import_fields( gcomp, cam_in, rc=rc ) if (ChkErr(rc,__LINE__,u_FILE_u)) return call cam_timestep_init() - call cam_run1 ( cam_in, cam_out ) + call cam_run1 () call export_fields( gcomp, model_mesh, model_clock, cam_out, rc=rc ) if (ChkErr(rc,__LINE__,u_FILE_u)) return else @@ -925,7 +923,7 @@ subroutine DataInitialize(gcomp, rc) call import_fields( gcomp, cam_in, restart_init=.true., rc=rc ) if (ChkErr(rc,__LINE__,u_FILE_u)) return call cam_timestep_init() - call cam_run1 ( cam_in, cam_out ) + call cam_run1 () call export_fields( gcomp, model_mesh, model_clock, cam_out, rc=rc ) if (ChkErr(rc,__LINE__,u_FILE_u)) return end if @@ -978,7 +976,7 @@ subroutine DataInitialize(gcomp, rc) !--------------------------------------------------------------- call cam_timestep_init() - call cam_run1 ( cam_in, cam_out ) + call cam_run1 () call NUOPC_CompAttributeSet(gcomp, name="InitializeDataComplete", value="true", rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return @@ -1154,15 +1152,15 @@ subroutine ModelAdvance(gcomp, rc) ! This includes the "physics_after_coupler" CCPP physics group. call t_startf ('CAM_run2') - call cam_run2( cam_out, cam_in ) + call cam_run2() call t_stopf ('CAM_run2') call t_startf ('CAM_run3') - call cam_run3( cam_out ) + call cam_run3() call t_stopf ('CAM_run3') call t_startf ('CAM_run4') - call cam_run4( cam_out, cam_in, rstwr, nlend, & + call cam_run4( rstwr, nlend, & yr_spec=yr_sync, mon_spec=mon_sync, day_spec=day_sync, sec_spec=tod_sync) call t_stopf ('CAM_run4') call cam_timestep_final(rstwr, nlend, do_ncdata_check=do_ncdata_check) @@ -1178,7 +1176,7 @@ subroutine ModelAdvance(gcomp, rc) ! This includes the "physics_before_coupler" CCPP physics group. call t_startf ('CAM_run1') - call cam_run1 ( cam_in, cam_out ) + call cam_run1 () call t_stopf ('CAM_run1') end do @@ -1451,7 +1449,7 @@ subroutine ModelFinalize(gcomp, rc) endif call cam_timestep_final(rstwr, nlend, do_ncdata_check=.false., do_history_write=.false.) - call cam_final(cam_out, cam_in) + call cam_final() if (masterproc) then write(iulog,F91) From b572d80f38a7326e032e4020bc48e72d93895824 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 15 Sep 2025 09:19:42 -0600 Subject: [PATCH 077/150] Update git-fleximod to v1.0.2 --- .lib/git-fleximod/git_fleximod/cli.py | 58 +++++++++------ .../git-fleximod/git_fleximod/git_fleximod.py | 18 +++-- .../git-fleximod/git_fleximod/gitinterface.py | 30 ++++++++ .lib/git-fleximod/git_fleximod/submodule.py | 74 +++++++++++++------ .lib/git-fleximod/pyproject.toml | 3 +- .lib/git-fleximod/tbump.toml | 2 +- .lib/git-fleximod/tests/conftest.py | 4 +- .lib/git-fleximod/tests/test_c_required.py | 13 ++++ 8 files changed, 147 insertions(+), 55 deletions(-) diff --git a/.lib/git-fleximod/git_fleximod/cli.py b/.lib/git-fleximod/git_fleximod/cli.py index 131466b9..b5a05494 100644 --- a/.lib/git-fleximod/git_fleximod/cli.py +++ b/.lib/git-fleximod/git_fleximod/cli.py @@ -1,38 +1,50 @@ from pathlib import Path -import argparse +import argparse, os, sys from git_fleximod import utils -__version__ = "0.9.4" +__version__ = "1.0.2" + +class CustomArgumentParser(argparse.ArgumentParser): + def print_help(self, file=None): + # First print the default help message + super().print_help(file) + + # Then append the contents of README.md + candidate_paths = [ + Path(sys.prefix) / "share" / "git_fleximod" / "README.md", + Path(__file__).resolve().parent.parent / "README.md", # fallback for dev + ] + for path in candidate_paths: + if os.path.exists(path): + with open(path) as f: + print( f.read(), file=file) + return + print( "README.md not found.", file=file) def find_root_dir(filename=".gitmodules"): """ finds the highest directory in tree which contains a file called filename """ - try: - root = utils.execute_subprocess(["git","rev-parse", "--show-toplevel"], - output_to_caller=True ).rstrip() - except: - d = Path.cwd() - root = Path(d.root) - dirlist = [] - dl = d - while dl != root: - dirlist.append(dl) - dl = dl.parent - dirlist.append(root) - dirlist.reverse() - - for dl in dirlist: - attempt = dl / filename - if attempt.is_file(): - return str(dl) - return None - return Path(root) + d = Path.cwd() + root = Path(d.root) + dirlist = [] + dl = d + while dl != root: + dirlist.append(dl) + dl = dl.parent + dirlist.append(root) + dirlist.reverse() + + for dl in dirlist: + attempt = dl / filename + if attempt.is_file(): + return str(dl) + return None def get_parser(): description = """ %(prog)s manages checking out groups of gitsubmodules with additional support for Earth System Models """ - parser = argparse.ArgumentParser( + parser = CustomArgumentParser( description=description, formatter_class=argparse.RawDescriptionHelpFormatter ) diff --git a/.lib/git-fleximod/git_fleximod/git_fleximod.py b/.lib/git-fleximod/git_fleximod/git_fleximod.py index 2c2601fa..b3c4fece 100755 --- a/.lib/git-fleximod/git_fleximod/git_fleximod.py +++ b/.lib/git-fleximod/git_fleximod/git_fleximod.py @@ -9,6 +9,7 @@ import shutil import logging import textwrap +import asyncio from git_fleximod import utils from git_fleximod import cli from git_fleximod.gitinterface import GitInterface @@ -218,10 +219,10 @@ def git_toplevelroot(root_dir, logger): _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") return superroot -def submodules_update(gitmodules, root_dir, requiredlist, force): - for name in gitmodules.sections(): +async def submodules_update(gitmodules, root_dir, requiredlist, force): + async def update_submodule(name, requiredlist, force): submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) - + _, needsupdate, localmods, testfails = submod.status() if not submod.fxrequired: submod.fxrequired = "AlwaysRequired" @@ -239,11 +240,11 @@ def submodules_update(gitmodules, root_dir, requiredlist, force): if "Optional" in fxrequired and "Optional" not in requiredlist: if fxrequired.startswith("Always"): print(f"Skipping optional component {name:>20}") - continue + return # continue to next submodule optional = "AlwaysOptional" in requiredlist if fxrequired in requiredlist: - submod.update() + await submod.update() repodir = os.path.join(root_dir, submod.path) if os.path.exists(os.path.join(repodir, ".gitmodules")): # recursively handle this checkout @@ -252,7 +253,10 @@ def submodules_update(gitmodules, root_dir, requiredlist, force): newrequiredlist = ["AlwaysRequired"] if optional: newrequiredlist.append("AlwaysOptional") - submodules_update(gitsubmodules, repodir, newrequiredlist, force=force) + await submodules_update(gitsubmodules, repodir, newrequiredlist, force=force) + + tasks = [update_submodule(name, requiredlist, force) for name in gitmodules.sections()] + await asyncio.gather(*tasks) def local_mods_output(): text = """\ @@ -346,7 +350,7 @@ def main(): sys.exit(f"No submodule components found, root_dir={root_dir}") retval = 0 if action == "update": - submodules_update(gitmodules, root_dir, fxrequired, force) + asyncio.run(submodules_update(gitmodules, root_dir, fxrequired, force)) elif action == "status": tfails, lmods, updates = submodules_status(gitmodules, root_dir, toplevel=True) if tfails + lmods + updates > 0: diff --git a/.lib/git-fleximod/git_fleximod/gitinterface.py b/.lib/git-fleximod/git_fleximod/gitinterface.py index fb20883c..022426d2 100644 --- a/.lib/git-fleximod/git_fleximod/gitinterface.py +++ b/.lib/git-fleximod/git_fleximod/gitinterface.py @@ -2,6 +2,7 @@ import sys from . import utils from pathlib import Path +import asyncio class GitInterface: def __init__(self, repo_path, logger): @@ -47,6 +48,16 @@ def _init_git_repo(self): command = ("git", "-C", str(self.repo_path), "init") utils.execute_subprocess(command) + def _git_operation_command(self, operation, args): + newargs = [] + for a in args: + # Do not use ssh interface + if isinstance(a, str): + a = a.replace("git@github.com:", "https://github.com/") + newargs.append(a) + + return self._git_command(operation, *newargs) + # pylint: disable=unused-argument def git_operation(self, operation, *args, **kwargs): newargs = [] @@ -66,6 +77,25 @@ def git_operation(self, operation, *args, **kwargs): else: return 0, command + # pylint: disable=unused-argument + async def git_operation_async(self, operation, *args, **kwargs): + command = self._git_operation_command(operation, args) + if isinstance(command, list): + try: + process = await asyncio.create_subprocess_exec( + *command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await process.communicate() + status = process.returncode + output = stdout.decode().strip() if stdout else stderr.decode().strip() + return status, output + except Exception as e: + sys.exit(e) + else: + return 0, command + def config_get_value(self, section, name): if self._use_module: config = self.repo.config_reader() diff --git a/.lib/git-fleximod/git_fleximod/submodule.py b/.lib/git-fleximod/git_fleximod/submodule.py index 75d9dd4e..4c6c3acc 100644 --- a/.lib/git-fleximod/git_fleximod/submodule.py +++ b/.lib/git-fleximod/git_fleximod/submodule.py @@ -119,27 +119,25 @@ def status(self): atag = atag[:-1] if atag == self.fxtag: break - - - #print(f"line is {line} ahash is {ahash} atag is {atag} {parts}") - # atag = git.git_operation("describe", "--tags", "--always") - # ahash = git.git_operation("rev-list", "HEAD").partition("\n")[0] - recurse = False if rurl != self.url: remote = self._add_remote(git) git.git_operation("fetch", remote) + # Asked for a tag and found that tag if self.fxtag and atag == self.fxtag: result = f" {self.name:>20} at tag {self.fxtag}" recurse = True testfails = False + # Asked for and found a hash elif self.fxtag and (ahash[: len(self.fxtag)] == self.fxtag or (self.fxtag.find(ahash)==0)): result = f" {self.name:>20} at hash {ahash}" recurse = True testfails = False + # Asked for and found a hash elif atag == ahash: result = f" {self.name:>20} at hash {ahash}" recurse = True + # Did not find requested tag or hash elif self.fxtag: result = f"s {self.name:>20} {atag} {ahash} is out of sync with .gitmodules {self.fxtag}" testfails = True @@ -284,17 +282,18 @@ def sparse_checkout(self): if not os.path.isdir(infodir): os.makedirs(infodir) gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout")) - if os.path.isfile(gitsparse): - self.logger.warning( - "submodule {} is already initialized {}".format(self.name, rootdotgit) - ) - return - - with utils.pushd(sprep_repo): + if os.path.isfile(gitsparse): + self.logger.warning( + "submodule {} is already initialized {}".format(self.name, rootdotgit) + ) + os.remove(gitsparse) + if os.path.isfile(self.fxsparse): - shutil.copy(self.fxsparse, gitsparse) - + else: + self.logger.warning( + "submodule {} could not find {}".format(self.name, self.fxsparse) + ) # Finally checkout the repo sprepo_git.git_operation("fetch", "origin", "--tags") @@ -303,11 +302,18 @@ def sparse_checkout(self): print(f"Error checking out {self.name:>20} at {self.fxtag}") else: print(f"Successfully checked out {self.name:>20} at {self.fxtag}") + status,f = sprepo_git.git_operation("status") + # Restore any files deleted from sandbox + for line in f.splitlines(): + if "deleted:" in line: + deleted_file = line.split("deleted:")[1].strip() + sprepo_git.git_operation("checkout", deleted_file) + rgit.config_set_value('submodule.' + self.name, "active", "true") rgit.config_set_value('submodule.' + self.name, "url", self.url) rgit.config_set_value('submodule.' + self.name, "path", self.path) - def update(self): + async def update(self): """ Updates the submodule to the latest or specified version. @@ -341,6 +347,9 @@ def update(self): # Look for a .gitmodules file in the newly checkedout repo if self.fxsparse: print(f"Sparse checkout {self.name} fxsparse {self.fxsparse}") + if not os.path.isfile(self.fxsparse): + self.logger.info("Submodule {} fxsparse file not found".format(self.name)) + self.sparse_checkout() else: if not repo_exists and self.url: @@ -378,19 +387,26 @@ def update(self): git.git_operation("submodule", "add", "--name", self.name, "--", self.url, self.path) if not repo_exists: - git.git_operation("submodule", "update", "--init", "--", self.path) + git.git_operation("submodule", "init", "--", self.path) + await git.git_operation_async("submodule", "update", "--", self.path) if self.fxtag: smgit = GitInterface(repodir, self.logger) newremote = self._add_remote(smgit) # Trying to distingush a tag from a hash - allowed = set(string.digits + 'abcdef') + allowed = set(string.digits + 'abcdef') + status = 0 if not set(self.fxtag) <= allowed: # This is a tag tag = f"refs/tags/{self.fxtag}:refs/tags/{self.fxtag}" - smgit.git_operation("fetch", newremote, tag) - smgit.git_operation("checkout", self.fxtag) - + status,_ = smgit.git_operation("fetch", newremote, tag) + if status == 0: + status,_ = smgit.git_operation("checkout", self.fxtag) + if status: + utils.fatal_error( + f"Failed to checkout {self.name} at tag or hash {self.fxtag} from {repodir}" + ) + if not os.path.exists(os.path.join(repodir, ".git")): utils.fatal_error( f"Failed to checkout {self.name} {repo_exists} {repodir} {self.path}" @@ -408,6 +424,18 @@ def update(self): if fxtag and fxtag not in tags: git.git_operation("fetch", newremote, "--tags") status, atag = git.git_operation("describe", "--tags", "--always") + status, files = git.git_operation("diff", "--name-only", "-z") + modfiles = [] + moddirs = [] + if files: + for f in files.split('\0'): + if f: + if os.path.exists(f): + git.git_operation("checkout",f) + elif os.path.isdir(f): + moddirs.append(f) + else: + modfiles.append(f) if fxtag and fxtag != atag: try: status, _ = git.git_operation("checkout", fxtag) @@ -419,6 +447,10 @@ def update(self): elif not fxtag: print(f"No fxtag found for submodule {self.name:>20}") + elif modfiles: + print(f"{self.name:>20} has modified files: {modfiles}") + elif moddirs: + print(f"{self.name:>20} has modified directories: {moddirs}") else: print(f"{self.name:>20} up to date.") diff --git a/.lib/git-fleximod/pyproject.toml b/.lib/git-fleximod/pyproject.toml index 65924ff9..029fd65e 100644 --- a/.lib/git-fleximod/pyproject.toml +++ b/.lib/git-fleximod/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "git-fleximod" -version = "0.9.4" +version = "1.0.2" description = "Extended support for git-submodule and git-sparse-checkout" authors = ["Jim Edwards "] maintainers = ["Jim Edwards "] @@ -11,6 +11,7 @@ keywords = ["git", "submodule", "sparse-checkout"] packages = [ { include = "git_fleximod"}, { include = "doc"}, +{ include = "README.md"}, ] [tool.poetry.scripts] diff --git a/.lib/git-fleximod/tbump.toml b/.lib/git-fleximod/tbump.toml index be0b799d..f6fe1e88 100644 --- a/.lib/git-fleximod/tbump.toml +++ b/.lib/git-fleximod/tbump.toml @@ -2,7 +2,7 @@ github_url = "https://github.com/jedwards4b/git-fleximod/" [version] -current = "0.9.4" +current = "1.0.2" # Example of a semver regexp. # Make sure this matches current_version before diff --git a/.lib/git-fleximod/tests/conftest.py b/.lib/git-fleximod/tests/conftest.py index 1dd1b86f..44d28e17 100644 --- a/.lib/git-fleximod/tests/conftest.py +++ b/.lib/git-fleximod/tests/conftest.py @@ -32,7 +32,7 @@ def logger(): "submodule_name": "test_optional", "status1" : "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", "status2" : "test_optional at tag MPIserial_2.4.0", - "status3" : "test_optional not checked out, out of sync at tag MPIserial_2.5.1, expected tag is MPIserial_2.4.0 (optional)", + "status3" : "test_optional not checked out, out of sync at tag MPIserial_2.5.4, expected tag is MPIserial_2.4.0 (optional)", "status4" : "test_optional at tag MPIserial_2.4.0", "gitmodules_content": """ [submodule "test_optional"] @@ -46,7 +46,7 @@ def logger(): "submodule_name": "test_alwaysoptional", "status1" : "test_alwaysoptional MPIserial_2.3.0 is out of sync with .gitmodules e5cf35c", "status2" : "test_alwaysoptional at hash e5cf35c", - "status3" : "out of sync at tag MPIserial_2.5.1, expected tag is e5cf35c", + "status3" : "out of sync at tag MPIserial_2.5.4, expected tag is e5cf35c", "status4" : "test_alwaysoptional at hash e5cf35c", "gitmodules_content": """ [submodule "test_alwaysoptional"] diff --git a/.lib/git-fleximod/tests/test_c_required.py b/.lib/git-fleximod/tests/test_c_required.py index 89ab8d29..2ac66145 100644 --- a/.lib/git-fleximod/tests/test_c_required.py +++ b/.lib/git-fleximod/tests/test_c_required.py @@ -1,4 +1,5 @@ import pytest +import re from pathlib import Path def test_required(git_fleximod, test_repo, shared_repos): @@ -28,3 +29,15 @@ def test_required(git_fleximod, test_repo, shared_repos): assert result.returncode == 0 status = git_fleximod(test_repo, f"status {repo_name}") assert shared_repos["status4"] in status.stdout + + text = file_path.read_text() + new_value = "somethingelse" + pattern = r"(^\s*fxtag\s*=\s*).*$" + replacement = r"\1" + new_value + new_text = re.sub(pattern, replacement, text, flags=re.MULTILINE) + + # Write updated content back to file + file_path.write_text(new_text) + + result = git_fleximod(test_repo, f"update {repo_name}") + assert f'fatal: couldn\'t find remote ref' in result.stderr or 'error: pathspec \'somethingelse\' did not match any file(s) known to git' in result.stderr From 5a32d0d8337c86553185e61a3453246379e33fd8 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 15 Sep 2025 09:47:53 -0600 Subject: [PATCH 078/150] Update externals to match cam6_4_115. --- .gitmodules | 22 +++++++++++----------- ccs_config | 2 +- cime | 2 +- components/cdeps | 2 +- components/cism | 2 +- components/clm | 2 +- components/cmeps | 2 +- components/mizuRoute | 2 +- components/mosart | 2 +- components/rtm | 2 +- libraries/parallelio | 2 +- tools/CUPiD | 2 +- 12 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.gitmodules b/.gitmodules index 5b09d6ca..cc53ef5c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -32,13 +32,13 @@ [submodule "ccs_config"] path = ccs_config url = https://github.com/ESMCI/ccs_config_cesm.git - fxtag = ccs_config_cesm1.0.53 + fxtag = ccs_config_cesm1.0.59 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git [submodule "cdeps"] path = components/cdeps url = https://github.com/ESCOMP/CDEPS.git - fxtag = cdeps1.0.73 + fxtag = cdeps1.0.80 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git [submodule "cice"] @@ -50,25 +50,25 @@ [submodule "cime"] path = cime url = https://github.com/ESMCI/cime - fxtag = cime6.1.113 + fxtag = cime6.1.122 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESMCI/cime [submodule "cism"] path = components/cism url = https://github.com/ESCOMP/CISM-wrapper - fxtag = cismwrap_2_2_006 + fxtag = cismwrap_2_2_010 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CISM-wrapper [submodule "clm"] path = components/clm url = https://github.com/ESCOMP/CTSM - fxtag = ctsm5.3.029 + fxtag = alpha-ctsm5.4.CMIP7.09.ctsm5.3.068 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CTSM [submodule "cmeps"] path = components/cmeps url = https://github.com/ESCOMP/CMEPS.git - fxtag = cmeps1.0.47 + fxtag = cmeps1.1.16 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git [submodule "fms"] @@ -80,25 +80,25 @@ [submodule "mizuRoute"] path = components/mizuRoute url = https://github.com/ESCOMP/mizuRoute - fxtag = cesm-coupling.n03_v2.2.0 + fxtag = v3.0.0 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute [submodule "mosart"] path = components/mosart url = https://github.com/ESCOMP/MOSART - fxtag = mosart1.1.08 + fxtag = mosart1.1.12 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/MOSART [submodule "parallelio"] path = libraries/parallelio url = https://github.com/NCAR/ParallelIO - fxtag = pio2_6_4 + fxtag = pio2_6_6 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/NCAR/ParallelIO [submodule "rtm"] path = components/rtm url = https://github.com/ESCOMP/RTM - fxtag = rtm1_0_86 + fxtag = rtm1_0_89 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/RTM [submodule "share"] @@ -110,6 +110,6 @@ [submodule "tools/CUPiD"] path = tools/CUPiD url = https://github.com/NCAR/CUPiD.git - fxtag = v0.2.1 + fxtag = v0.3.1 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/NCAR/CUPiD.git diff --git a/ccs_config b/ccs_config index 8103ad41..028bbd77 160000 --- a/ccs_config +++ b/ccs_config @@ -1 +1 @@ -Subproject commit 8103ad41d9657950ff8e387524a3df1e84bcda9e +Subproject commit 028bbd779bf989cab8767dd0e7135126429b4531 diff --git a/cime b/cime index 1616f816..16f1fb44 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit 1616f81668cd061597487338980ab2608b5c9034 +Subproject commit 16f1fb44607037e5a1dfe769a69bad366baf7d12 diff --git a/components/cdeps b/components/cdeps index 9e57b278..692150fc 160000 --- a/components/cdeps +++ b/components/cdeps @@ -1 +1 @@ -Subproject commit 9e57b278de18cf69351bc693fe1988143e859fa7 +Subproject commit 692150fc020e31235784634dd3eab4e78629bb82 diff --git a/components/cism b/components/cism index 0a4e6cfe..84767787 160000 --- a/components/cism +++ b/components/cism @@ -1 +1 @@ -Subproject commit 0a4e6cfe8828cd28cdab06f83a7c1eb464c1ca8c +Subproject commit 84767787b78a3547497cd3f1b452e775cdc56f95 diff --git a/components/clm b/components/clm index be65fad3..09911759 160000 --- a/components/clm +++ b/components/clm @@ -1 +1 @@ -Subproject commit be65fad30a741fc594bca9e1221ffe987e7bf271 +Subproject commit 0991175998c94291f3d544169e3a4acb71b0b7f1 diff --git a/components/cmeps b/components/cmeps index dfe357ec..a0735ecd 160000 --- a/components/cmeps +++ b/components/cmeps @@ -1 +1 @@ -Subproject commit dfe357ec28da465b1d0e435594c6bd6700ce62f3 +Subproject commit a0735ecdb5a3e1150797a8f05ded6e4b6a271b41 diff --git a/components/mizuRoute b/components/mizuRoute index 362bee32..07dfd26a 160000 --- a/components/mizuRoute +++ b/components/mizuRoute @@ -1 +1 @@ -Subproject commit 362bee329bd6bf1fd45c8f36e006b9c4294bb8ca +Subproject commit 07dfd26a17acb914b9036f59a8c6237990da09f2 diff --git a/components/mosart b/components/mosart index 00a87c90..0faa624a 160000 --- a/components/mosart +++ b/components/mosart @@ -1 +1 @@ -Subproject commit 00a87c9084af1af0d2b14d14e3d432f6808681f9 +Subproject commit 0faa624a59b5ce677c40ed64f1ca65bb4997db8d diff --git a/components/rtm b/components/rtm index 26e96f50..1165b4ff 160000 --- a/components/rtm +++ b/components/rtm @@ -1 +1 @@ -Subproject commit 26e96f500b9500b32a870db20eed6b1bd37587ea +Subproject commit 1165b4ff0c015369d350478c5833cf366acada9f diff --git a/libraries/parallelio b/libraries/parallelio index a8446176..1d516b82 160000 --- a/libraries/parallelio +++ b/libraries/parallelio @@ -1 +1 @@ -Subproject commit a844617662cbb2b1a6445bd7560f9095ccefd180 +Subproject commit 1d516b82768ef9d2cade524724c38ab63fbe08e4 diff --git a/tools/CUPiD b/tools/CUPiD index e3d16bf4..3d953177 160000 --- a/tools/CUPiD +++ b/tools/CUPiD @@ -1 +1 @@ -Subproject commit e3d16bf49184cd6dbb18c17072031b974605c8e5 +Subproject commit 3d9531779b6f3e4c26059b10ba67c0f4477684e5 From f367b495bf3589dba899cd6c86f884e82afa009d Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 16 Sep 2025 11:49:23 -0600 Subject: [PATCH 079/150] Add debugging output to git-fleximod. --- .lib/git-fleximod/git_fleximod/submodule.py | 46 +++++++++++---------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/.lib/git-fleximod/git_fleximod/submodule.py b/.lib/git-fleximod/git_fleximod/submodule.py index 4c6c3acc..d5fbe240 100644 --- a/.lib/git-fleximod/git_fleximod/submodule.py +++ b/.lib/git-fleximod/git_fleximod/submodule.py @@ -27,7 +27,7 @@ def __init__(self, root_dir, name, path, url, fxtag=None, fxurl=None, fxsparse=N """ self.name = name self.root_dir = root_dir - self.path = path + self.path = path self.url = url self.fxurl = fxurl self.fxtag = fxtag @@ -37,14 +37,14 @@ def __init__(self, root_dir, name, path, url, fxtag=None, fxurl=None, fxsparse=N else: self.fxrequired = "AlwaysRequired" self.logger = logger - + def status(self): """ Checks the status of the submodule and returns 4 parameters: - result (str): The status of the submodule. - needsupdate (bool): An indicator if the submodule needs to be updated. - localmods (bool): An indicator if the submodule has local modifications. - - testfails (bool): An indicator if the submodule has failed a test, this is used for testing purposes. + - testfails (bool): An indicator if the submodule has failed a test, this is used for testing purposes. """ smpath = os.path.join(self.root_dir, self.path) @@ -54,7 +54,7 @@ def status(self): ahash = None optional = "" if "Optional" in self.fxrequired: - optional = " (optional)" + optional = " (optional)" required = None level = None if not os.path.exists(os.path.join(smpath, ".git")): @@ -63,7 +63,7 @@ def status(self): status, tags = rootgit.git_operation("ls-remote", "--tags", self.url) status, result = rootgit.git_operation("submodule","status",smpath) result = result.split() - + if result: ahash = result[0][1:] hhash = None @@ -102,7 +102,7 @@ def status(self): result = f"e {self.name:>20} has no associated remote" testfails = True needsupdate = True - return result, needsupdate, localmods, testfails + return result, needsupdate, localmods, testfails status, rurl = git.git_operation("ls-remote","--get-url") status, lines = git.git_operation("log", "--pretty=format:\"%h %d\"") line = lines.partition('\n')[0] @@ -148,7 +148,7 @@ def status(self): else: result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {ahash}" testfails = False - + status, output = git.git_operation("status", "--ignore-submodules", "-uno") if "nothing to commit" not in output: localmods = True @@ -156,7 +156,7 @@ def status(self): # print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}") return result, needsupdate, localmods, testfails - + def _add_remote(self, git): """ Adds a new remote to the submodule if it does not already exist. @@ -170,7 +170,7 @@ def _add_remote(self, git): Returns: str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL. - """ + """ status, remotes = git.git_operation("remote", "-v") remotes = remotes.splitlines() upstream = None @@ -212,7 +212,7 @@ def sparse_checkout(self): Returns: None - """ + """ self.logger.info("Called sparse_checkout for {}".format(self.name)) rgit = GitInterface(self.root_dir, self.logger) status, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") @@ -250,7 +250,7 @@ def sparse_checkout(self): sprepo_git.config_set_value("core", "sparseCheckout", "true") # set the repository remote - + self.logger.info("Setting remote origin in {}/{}".format(self.root_dir, self.path)) status, remotes = sprepo_git.git_operation("remote", "-v") if self.url not in remotes: @@ -287,7 +287,7 @@ def sparse_checkout(self): "submodule {} is already initialized {}".format(self.name, rootdotgit) ) os.remove(gitsparse) - + if os.path.isfile(self.fxsparse): shutil.copy(self.fxsparse, gitsparse) else: @@ -376,7 +376,7 @@ async def update(self): shutil.rmtree(os.path.join(repodir, ".git")) else: shutil.move(os.path.join(repodir, ".git"), newpath) - + with open(os.path.join(repodir, ".git"), "w") as f: f.write("gitdir: " + os.path.relpath(newpath, start=repodir)) @@ -384,13 +384,13 @@ async def update(self): parent = os.path.dirname(repodir) if not os.path.isdir(parent): os.makedirs(parent) - git.git_operation("submodule", "add", "--name", self.name, "--", self.url, self.path) + git.git_operation("submodule", "add", "--name", self.name, "--", self.url, self.path) if not repo_exists: git.git_operation("submodule", "init", "--", self.path) await git.git_operation_async("submodule", "update", "--", self.path) - if self.fxtag: + if self.fxtag: smgit = GitInterface(repodir, self.logger) newremote = self._add_remote(smgit) # Trying to distingush a tag from a hash @@ -399,19 +399,23 @@ async def update(self): if not set(self.fxtag) <= allowed: # This is a tag tag = f"refs/tags/{self.fxtag}:refs/tags/{self.fxtag}" - status,_ = smgit.git_operation("fetch", newremote, tag) + #status,_ = smgit.git_operation("fetch", newremote, tag) + status, jn = smgit.git_operation("fetch", newremote, tag) + print("Successfully fetch tag? -JN", status, jn, self.fxtag) #DEBUG -JN if status == 0: - status,_ = smgit.git_operation("checkout", self.fxtag) + #status,_ = smgit.git_operation("checkout", self.fxtag) + status,jn = smgit.git_operation("checkout", self.fxtag) + print("Successfully checkout tag? -JN", status, jn, self.fxtag) #DEBUG -JN if status: utils.fatal_error( f"Failed to checkout {self.name} at tag or hash {self.fxtag} from {repodir}" ) - + if not os.path.exists(os.path.join(repodir, ".git")): utils.fatal_error( f"Failed to checkout {self.name} {repo_exists} {repodir} {self.path}" ) - + if os.path.exists(os.path.join(self.path, ".git")): submoddir = os.path.join(self.root_dir, self.path) @@ -443,7 +447,7 @@ async def update(self): print(f"{self.name:>20} updated to {fxtag}") except Exception as error: print(error) - + elif not fxtag: print(f"No fxtag found for submodule {self.name:>20}") @@ -455,5 +459,5 @@ async def update(self): print(f"{self.name:>20} up to date.") - + return From 6cad15f0370cb1fce01e610b382b3fdeda250681 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 16 Sep 2025 13:15:26 -0600 Subject: [PATCH 080/150] Print out git commands in fleximod. --- .lib/git-fleximod/git_fleximod/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.lib/git-fleximod/git_fleximod/utils.py b/.lib/git-fleximod/git_fleximod/utils.py index c4f43d52..462e32c0 100644 --- a/.lib/git-fleximod/git_fleximod/utils.py +++ b/.lib/git-fleximod/git_fleximod/utils.py @@ -283,6 +283,7 @@ def execute_subprocess(commands, status_to_caller=False, output_to_caller=False) kwargs={"working_directory": cwd, "command": commands_str}, ) hanging_timer.start() + print("What are the commands? -JN", commands) #Debug -JN try: output = subprocess.check_output( commands, stderr=subprocess.STDOUT, universal_newlines=True From 3eaaf02eb77fdd7e4210e1895909bb36553ab7b0 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 16 Sep 2025 13:33:49 -0600 Subject: [PATCH 081/150] Remove debugging code. --- .lib/git-fleximod/git_fleximod/submodule.py | 8 ++------ .lib/git-fleximod/git_fleximod/utils.py | 1 - 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.lib/git-fleximod/git_fleximod/submodule.py b/.lib/git-fleximod/git_fleximod/submodule.py index d5fbe240..ba07c117 100644 --- a/.lib/git-fleximod/git_fleximod/submodule.py +++ b/.lib/git-fleximod/git_fleximod/submodule.py @@ -399,13 +399,9 @@ async def update(self): if not set(self.fxtag) <= allowed: # This is a tag tag = f"refs/tags/{self.fxtag}:refs/tags/{self.fxtag}" - #status,_ = smgit.git_operation("fetch", newremote, tag) - status, jn = smgit.git_operation("fetch", newremote, tag) - print("Successfully fetch tag? -JN", status, jn, self.fxtag) #DEBUG -JN + status,_ = smgit.git_operation("fetch", newremote, tag) if status == 0: - #status,_ = smgit.git_operation("checkout", self.fxtag) - status,jn = smgit.git_operation("checkout", self.fxtag) - print("Successfully checkout tag? -JN", status, jn, self.fxtag) #DEBUG -JN + status,_ = smgit.git_operation("checkout", self.fxtag) if status: utils.fatal_error( f"Failed to checkout {self.name} at tag or hash {self.fxtag} from {repodir}" diff --git a/.lib/git-fleximod/git_fleximod/utils.py b/.lib/git-fleximod/git_fleximod/utils.py index 462e32c0..c4f43d52 100644 --- a/.lib/git-fleximod/git_fleximod/utils.py +++ b/.lib/git-fleximod/git_fleximod/utils.py @@ -283,7 +283,6 @@ def execute_subprocess(commands, status_to_caller=False, output_to_caller=False) kwargs={"working_directory": cwd, "command": commands_str}, ) hanging_timer.start() - print("What are the commands? -JN", commands) #Debug -JN try: output = subprocess.check_output( commands, stderr=subprocess.STDOUT, universal_newlines=True From fcf380c31c9506a2e83b40666255e4cd9b1dab7e Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 19 Sep 2025 13:56:15 -0600 Subject: [PATCH 082/150] Update fleximod test and gitmodules file to match new CAM code/submodules. --- .github/workflows/fleximod_test.yaml | 6 ++---- .gitmodules | 6 +++--- ccs_config | 2 +- cime | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/fleximod_test.yaml b/.github/workflows/fleximod_test.yaml index c1112060..f6a766cb 100644 --- a/.github/workflows/fleximod_test.yaml +++ b/.github/workflows/fleximod_test.yaml @@ -13,7 +13,7 @@ jobs: uses: actions/checkout@v4 - id: run-fleximod run: | - $GITHUB_WORKSPACE/bin/git-fleximod update + $GITHUB_WORKSPACE/bin/git-fleximod update -o echo echo "Update complete, checking status" echo @@ -23,6 +23,4 @@ jobs: echo echo "Checking if git fleximod matches expected externals" echo - git diff --exit-code - - + git add . && git diff --exit-code && git diff --cached --exit-code diff --git a/.gitmodules b/.gitmodules index cc53ef5c..301b30ea 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/ESCOMP/atmospheric_physics - fxtag = 178c1833d72a4027c6162a6c11b52bbbdcb272b6 + fxtag = atmos_phys0_17_002 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] @@ -32,7 +32,7 @@ [submodule "ccs_config"] path = ccs_config url = https://github.com/ESMCI/ccs_config_cesm.git - fxtag = ccs_config_cesm1.0.59 + fxtag = ccs_config_cesm1.0.60 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git [submodule "cdeps"] @@ -50,7 +50,7 @@ [submodule "cime"] path = cime url = https://github.com/ESMCI/cime - fxtag = cime6.1.122 + fxtag = cime6.1.126 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESMCI/cime [submodule "cism"] diff --git a/ccs_config b/ccs_config index 028bbd77..ff41a396 160000 --- a/ccs_config +++ b/ccs_config @@ -1 +1 @@ -Subproject commit 028bbd779bf989cab8767dd0e7135126429b4531 +Subproject commit ff41a3963e325db3348d56da492c4e18535d34fc diff --git a/cime b/cime index 16f1fb44..c0681033 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit 16f1fb44607037e5a1dfe769a69bad366baf7d12 +Subproject commit c0681033bdfb71d10821b478fe562e825f28268b From 40cf4dfafd9d1d1c6951c790cdda36de2204ebc7 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 19 Sep 2025 14:56:30 -0600 Subject: [PATCH 083/150] Add missed atmospheric_physics submodule update. --- src/physics/ncar_ccpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 178c1833..cd1365da 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 178c1833d72a4027c6162a6c11b52bbbdcb272b6 +Subproject commit cd1365da0f23d0529a497757e64266ce60f746a3 From 45328eae205cdf9a4af205df12417cd6462ff812 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 23 Sep 2025 10:11:00 -0600 Subject: [PATCH 084/150] Update ccs_config submodule. --- .gitmodules | 2 +- ccs_config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 301b30ea..eae48dd9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -32,7 +32,7 @@ [submodule "ccs_config"] path = ccs_config url = https://github.com/ESMCI/ccs_config_cesm.git - fxtag = ccs_config_cesm1.0.60 + fxtag = ccs_config_cesm1.0.61 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git [submodule "cdeps"] diff --git a/ccs_config b/ccs_config index ff41a396..b20a207b 160000 --- a/ccs_config +++ b/ccs_config @@ -1 +1 @@ -Subproject commit ff41a3963e325db3348d56da492c4e18535d34fc +Subproject commit b20a207bc918b956b8dad44c14c4471aff9331ee From e340186420216b9f18b8909b1c36b87644f0ea8a Mon Sep 17 00:00:00 2001 From: peverwhee Date: Tue, 23 Sep 2025 16:41:00 -0600 Subject: [PATCH 085/150] commit latest host side changes --- .gitmodules | 2 +- src/control/cam_comp.F90 | 1 + src/control/cam_control_mod.F90 | 3 +++ src/data/registry.xml | 29 ++--------------------------- src/physics/ncar_ccpp | 2 +- 5 files changed, 8 insertions(+), 29 deletions(-) diff --git a/.gitmodules b/.gitmodules index de395262..16d8a682 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = 493db339ac9acfd73125d48e96a6cf8f138a0589 + fxtag = 4dc0b4cfa3136af462d17e2630f5c26355b06710 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/control/cam_comp.F90 b/src/control/cam_comp.F90 index 26b75d41..16a2f591 100644 --- a/src/control/cam_comp.F90 +++ b/src/control/cam_comp.F90 @@ -194,6 +194,7 @@ subroutine cam_init(caseid, ctitle, model_doi_url, & call mark_as_initialized('fractional_calendar_days_on_end_of_next_timestep') call mark_as_initialized('number_of_seconds_until_next_shortwave_radiation_timestep') call mark_as_initialized('next_calendar_day_to_perform_shortwave_radiation_for_surface_models') + call mark_as_initialized('number_of_wavelength_samples_of_spectrum') ! Read CAM namelists. filein = "atm_in" // trim(inst_suffix) diff --git a/src/control/cam_control_mod.F90 b/src/control/cam_control_mod.F90 index d135eea8..34ca3c3e 100644 --- a/src/control/cam_control_mod.F90 +++ b/src/control/cam_control_mod.F90 @@ -101,6 +101,7 @@ end subroutine cam_ctrl_init !--------------------------------------------------------------------------- subroutine cam_ctrl_set_orbit(eccen_in, obliqr_in, lambm0_in, mvelpp_in) + use phys_vars_init_check, only: mark_as_initialized real(r8), intent(in) :: eccen_in real(r8), intent(in) :: obliqr_in @@ -112,6 +113,8 @@ subroutine cam_ctrl_set_orbit(eccen_in, obliqr_in, lambm0_in, mvelpp_in) lambm0 = lambm0_in mvelpp = mvelpp_in + call mark_as_initialized('planet_orbital_eccentricity_factor') + end subroutine cam_ctrl_set_orbit end module cam_control_mod diff --git a/src/data/registry.xml b/src/data/registry.xml index 2f0eb6ad..a9eb8f11 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -1610,31 +1610,6 @@ horizontal_dimension vertical_layer_dimension DES pbuf_DES - - -1 - - - -1 - - - -1 - - - -1 - - - -1 - horizontal_dimension vertical_layer_dimension QRL pbuf_QRL - + units="count" type="integer"> 2 Date: Wed, 24 Sep 2025 21:47:55 -0400 Subject: [PATCH 086/150] Add registry fields to read in pbuf fields to support GW CAM7. --- src/data/registry.xml | 44 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/src/data/registry.xml b/src/data/registry.xml index 5decdf8f..ff58c776 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -127,17 +127,17 @@ horizontal_dimension vertical_layer_dimension - frontgf pbuf_frontgf + frontgf pbuf_FRONTGF horizontal_dimension vertical_layer_dimension - frontga pbuf_frontga + frontga pbuf_FRONTGA 273.15_kind_phys tpert pbuf_tpert + + + horizontal_dimension vertical_layer_dimension + pbuf_TTEND_CLUBB + + + horizontal_dimension vertical_interface_dimension + pbuf_UPWP_CLUBB + + + horizontal_dimension vertical_interface_dimension + pbuf_VPWP_CLUBB + + + + + horizontal_dimension vertical_layer_dimension + pbuf_VORT4GW + horizontal_dimension vertical_layer_dimension number_of_ccpp_constituents 1 + + horizontal_dimension vertical_layer_dimension + pbuf_TTEND_DP + Date: Thu, 25 Sep 2025 01:20:19 -0400 Subject: [PATCH 087/150] Fix ridge read; clubb pbuf input vars have _gw suffix --- src/data/registry.xml | 4 ++-- .../utils/gravity_wave_drag_ridge_read.F90 | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/data/registry.xml b/src/data/registry.xml index ff58c776..f41756bb 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -1350,14 +1350,14 @@ units="m2 s-2" type="real" kind="kind_phys" allocatable="allocatable"> horizontal_dimension vertical_interface_dimension - pbuf_UPWP_CLUBB + pbuf_UPWP_CLUBB_GW horizontal_dimension vertical_interface_dimension - pbuf_VPWP_CLUBB + pbuf_VPWP_CLUBB_GW diff --git a/src/physics/utils/gravity_wave_drag_ridge_read.F90 b/src/physics/utils/gravity_wave_drag_ridge_read.F90 index 3eb108bc..f22d08ab 100644 --- a/src/physics/utils/gravity_wave_drag_ridge_read.F90 +++ b/src/physics/utils/gravity_wave_drag_ridge_read.F90 @@ -129,6 +129,7 @@ subroutine gravity_wave_drag_ridge_read_file() use physconst, only: rearth use cam_initfiles, only: unset_path_str use physics_grid, only: ncol => columns_on_task + use phys_vars_init_check, only: mark_as_initialized ! Local variables type(file_desc_t), pointer :: fh_topo @@ -144,6 +145,7 @@ subroutine gravity_wave_drag_ridge_read_file() errflg = 0 has_gbxar_from_topo = .false. + call mark_as_initialized('number_of_ridges_in_ridge_gravity_wave_drag_tbd') ! Do we have meso-Beta file? if(bnd_topo /= unset_path_str) then @@ -246,6 +248,16 @@ subroutine gravity_wave_drag_ridge_read_file() if(.not. found) then call endrun(trim(subname) // ': ANGLL not found in input file') endif + + ! Mark variables as initialized so they are not read from ic file. + call mark_as_initialized('grid_box_area_for_beta_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('isotropic_variance_for_beta_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('isotropic_weight_for_beta_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('ridge_half_width_for_beta_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('ridge_length_for_beta_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('ridge_obstacle_height_for_beta_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('ridge_anisotropy_for_beta_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('ridge_clockwise_angle_from_north_for_beta_ridge_gravity_wave_drag_tbd') endif ! Do we have meso-Gamma file? @@ -335,6 +347,13 @@ subroutine gravity_wave_drag_ridge_read_file() call cam_pio_closefile(fh_rdggm) deallocate(fh_rdggm) + ! Mark variables as initialized so they are not read from ic file. + call mark_as_initialized('grid_box_area_for_gamma_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('ridge_half_width_for_gamma_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('ridge_length_for_gamma_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('ridge_obstacle_height_for_gamma_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('ridge_anisotropy_for_gamma_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('ridge_clockwise_angle_from_north_for_gamma_ridge_gravity_wave_drag_tbd') endif end subroutine gravity_wave_drag_ridge_read_file From f9d95392a4ededaa909882c2693d157b80ebfaac Mon Sep 17 00:00:00 2001 From: peverwhee Date: Wed, 8 Oct 2025 11:58:13 -0600 Subject: [PATCH 088/150] read coszrs from file; add dycore name to runtime_obj --- .gitmodules | 2 +- src/control/cam_comp.F90 | 4 +-- src/control/runtime_obj.F90 | 17 ++++++++++++ src/data/registry.xml | 7 +++++ src/dynamics/mpas/dyn_comp.F90 | 3 +-- src/dynamics/mpas/dyn_comp_impl.F90 | 8 +++--- src/dynamics/none/dyn_comp.F90 | 6 ++--- src/dynamics/se/dyn_comp.F90 | 8 +++--- src/physics/utils/orbital_data.F90 | 41 +++++++++++++++++------------ src/physics/utils/orbital_data.meta | 6 ----- 10 files changed, 65 insertions(+), 37 deletions(-) diff --git a/.gitmodules b/.gitmodules index f2841ebb..c994a9f5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = 4dc0b4cfa3136af462d17e2630f5c26355b06710 + fxtag = aca8edf7c6c6d20902f717d4af5c6d85cbbd81aa fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/control/cam_comp.F90 b/src/control/cam_comp.F90 index 16a2f591..9df8a387 100644 --- a/src/control/cam_comp.F90 +++ b/src/control/cam_comp.F90 @@ -236,7 +236,7 @@ subroutine cam_init(caseid, ctitle, model_doi_url, & if (initial_run_in) then - call dyn_init(cam_runtime_opts, dyn_in, dyn_out) + call dyn_init(dyn_in, dyn_out) else @@ -312,7 +312,7 @@ subroutine cam_timestep_init() ! Update the orbital data call orbital_data_advance(calday, lat_rad, lon_rad, use_rad_uniform_angle, & - rad_uniform_angle, dt_avg) + rad_uniform_angle, dt_avg) ! Update timestep flags in physics state is_first_timestep = is_first_step() diff --git a/src/control/runtime_obj.F90 b/src/control/runtime_obj.F90 index 03041f46..b5c388f1 100644 --- a/src/control/runtime_obj.F90 +++ b/src/control/runtime_obj.F90 @@ -24,6 +24,7 @@ module runtime_obj !! type, public :: runtime_options character(len=CS), private :: phys_suite = unset_str + character(len=CS), private :: dycore = unset_str character(len=16), private :: waccmx_opt = unset_str ! use_gw_front: Frontogenesis logical, private :: use_gw_front = .false. @@ -34,12 +35,14 @@ module runtime_obj contains ! General runtime access procedure, public :: physics_suite + procedure, public :: get_dycore procedure, public :: suite_as_list ! Runtime parameters of interest to dycore procedure, public :: waccmx_on procedure, public :: waccmx_option procedure, public :: gw_front procedure, public :: gw_front_igw + procedure, public :: set_dycore procedure, public :: update_thermodynamic_variables end type runtime_options @@ -58,6 +61,12 @@ pure character(len=CS) function physics_suite(self) physics_suite = trim(self%phys_suite) end function physics_suite + pure character(len=CS) function get_dycore(self) + class(runtime_options), intent(in) :: self + + get_dycore = trim(self%dycore) + end function get_dycore + pure function suite_as_list(self) result(slist) class(runtime_options), intent(in) :: self character(len=CS) :: slist(1) @@ -100,6 +109,14 @@ pure logical function update_thermodynamic_variables(self) end function update_thermodynamic_variables + subroutine set_dycore(self, dycore_in) + class(runtime_options), intent(inout) :: self + character(len=*), intent(in) :: dycore_in + + self%dycore = trim(dycore_in) + + end subroutine set_dycore + subroutine cam_set_runtime_opts(phys_suite, waccmx_opt, & gw_front, gw_front_igw) use cam_abortutils, only: endrun diff --git a/src/data/registry.xml b/src/data/registry.xml index a9eb8f11..c6c77a3d 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -1529,6 +1529,13 @@ units="s" type="real" kind="kind_phys"> 0._kind_phys + + horizontal_dimension + COSZRS pbuf_COSZRS + (KCW, 2024-05-28) ! ! Called by `cam_init` in `src/control/cam_comp.F90`. - module subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) + module subroutine dyn_init(dyn_in, dyn_out) ! Module(s) from CAM-SIMA. use cam_abortutils, only: check_allocate use cam_constituents, only: const_name, const_is_water_species, num_advected, readtrace @@ -135,14 +135,13 @@ module subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) use cam_pio_utils, only: clean_iodesc_list use dyn_coupling, only: dyn_exchange_constituent_states use inic_analytic, only: analytic_ic_active - use runtime_obj, only: runtime_options + use runtime_obj, only: runtime_options, cam_runtime_opts use time_manager, only: get_step_size ! Module(s) from CCPP. use phys_vars_init_check, only: std_name_len ! Module(s) from external libraries. use pio, only: file_desc_t - type(runtime_options), intent(in) :: cam_runtime_opts type(dyn_import_t), intent(in) :: dyn_in type(dyn_export_t), intent(in) :: dyn_out @@ -155,6 +154,9 @@ module subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) type(file_desc_t), pointer :: pio_init_file type(file_desc_t), pointer :: pio_topo_file + ! Set dycore name in runtime object + call cam_runtime_opts%set_dycore('mpas') + call dyn_debug_print(debugout_debug, subname // ' entered') nullify(pio_init_file) diff --git a/src/dynamics/none/dyn_comp.F90 b/src/dynamics/none/dyn_comp.F90 index 9ecb3022..2844512e 100644 --- a/src/dynamics/none/dyn_comp.F90 +++ b/src/dynamics/none/dyn_comp.F90 @@ -51,17 +51,17 @@ end subroutine dyn_register !============================================================================== - subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) - use runtime_obj, only: runtime_options + subroutine dyn_init(dyn_in, dyn_out) + use runtime_obj, only: runtime_options, cam_runtime_opts ! Null dycore, no action ! Dummy arguments: - type(runtime_options), intent(in) :: cam_runtime_opts type(dyn_import_t), intent(out) :: dyn_in type(dyn_export_t), intent(out) :: dyn_out ! Note: dynamical core energy formula is set in dyn_grid based on dynamical core ! that provided the initial conditions file + call cam_runtime_opts%set_dycore('null') end subroutine dyn_init diff --git a/src/dynamics/se/dyn_comp.F90 b/src/dynamics/se/dyn_comp.F90 index bf5bfcab..8fac2ad5 100644 --- a/src/dynamics/se/dyn_comp.F90 +++ b/src/dynamics/se/dyn_comp.F90 @@ -556,8 +556,8 @@ end subroutine dyn_readnl !========================================================================================= -subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) - use runtime_obj, only: runtime_options +subroutine dyn_init(dyn_in, dyn_out) + use runtime_obj, only: runtime_options, cam_runtime_opts use dyn_grid, only: elem, fvm use cam_pio_utils, only: clean_iodesc_list use air_composition, only: thermodynamic_active_species_num, thermodynamic_active_species_idx @@ -586,7 +586,6 @@ subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) use control_mod, only: vert_remap_uvTq_alg, vert_remap_tracer_alg ! Dummy arguments: - type(runtime_options), intent(in) :: cam_runtime_opts type(dyn_import_t), intent(out) :: dyn_in type(dyn_export_t), intent(out) :: dyn_out @@ -654,6 +653,9 @@ subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) dycore_energy_consistency_adjust = .true. call mark_as_initialized("flag_for_dycore_energy_consistency_adjustment") + ! Set name of dycore in runtime object + call cam_runtime_opts%set_dycore('se') + ! Now allocate and set condenstate vars allocate(cnst_name_gll(qsize), stat=iret) ! constituent names for gll tracers call check_allocate(iret, subname, 'cnst_name_gll(qsize)', & diff --git a/src/physics/utils/orbital_data.F90 b/src/physics/utils/orbital_data.F90 index 9d18fbd0..393f5247 100644 --- a/src/physics/utils/orbital_data.F90 +++ b/src/physics/utils/orbital_data.F90 @@ -9,6 +9,8 @@ module orbital_data use ccpp_kinds, only: kind_phys use shr_orb_mod, only: FILL_R8 => SHR_ORB_UNDEF_REAL + use physics_types, only: coszrs_rad + use runtime_obj, only: cam_runtime_opts implicit none private @@ -21,7 +23,7 @@ module orbital_data real(kind_phys), protected, public :: solar_declination = FILL_R8 ! Solar declination angle [radians] real(kind_phys), protected, public :: earth_sun_distance = FILL_R8 ! Earth-sun distance [AU] real(kind_phys), allocatable, protected, public :: solar_zenith_angle(:) ! Solar zenith angle (column) [radians] - real(kind_phys), allocatable, protected, public :: coszrs_rad(:) ! Cosine of solar zenith angle (column) for radiation [radians] + !real(kind_phys), allocatable, protected, public :: coszrs_rad(:) ! Cosine of solar zenith angle (column) for radiation [radians] ! Local parameters character(len=*), parameter :: module_name = '(orbital_data)' @@ -51,11 +53,11 @@ subroutine orbital_data_init(number_of_columns) call check_allocate(error_code, subroutine_name, & 'solar_zenith_angle(number_of_columns)', & file=__FILE__, line=__LINE__) - allocate(coszrs_rad(number_of_columns), source=FILL_R8, & - stat=error_code) - call check_allocate(error_code, subroutine_name, & - 'coszrs_rad(number_of_columns)', & - file=__FILE__, line=__LINE__) + !allocate(coszrs_rad(number_of_columns), source=FILL_R8, & + ! stat=error_code) + !call check_allocate(error_code, subroutine_name, & + ! 'coszrs_rad(number_of_columns)', & + ! file=__FILE__, line=__LINE__) end subroutine orbital_data_init @@ -72,6 +74,7 @@ subroutine orbital_data_advance(calendar_day, latitudes, longitudes, use_rad_uni use shr_orb_mod, only: shr_orb_decl, shr_orb_cosz use cam_control_mod, only: eccen, mvelpp, lambm0, obliqr + use phys_vars_init_check, only: mark_as_initialized real(kind_phys), intent(in) :: calendar_day ! Fractional Julian calendar day (1.xx to 365.xx) real(kind_phys), intent(in) :: latitudes(:) ! Centered latitude (column) [radians] @@ -93,17 +96,21 @@ subroutine orbital_data_advance(calendar_day, latitudes, longitudes, use_rad_uni end do ! Compute the cosine of solar zenith angle for radiation [radians] - if (use_rad_uniform_angle) then - do i = 1, size(latitudes) - coszrs_rad(i) = shr_orb_cosz(calendar_day, latitudes(i), longitudes(i), & - solar_declination, dt_avg, uniform_angle=rad_uniform_angle) - end do - else - do i = 1, size(latitudes) - ! if dt_avg /= 0, it triggers using avg coszrs - coszrs_rad(i) = shr_orb_cosz(calendar_day, latitudes(i), longitudes(i), & - solar_declination, dt_avg) - end do + ! Don't compute this if we're running with snapshot files + if (trim(cam_runtime_opts%get_dycore()) /= 'null') then + if (use_rad_uniform_angle) then + do i = 1, size(latitudes) + coszrs_rad(i) = shr_orb_cosz(calendar_day, latitudes(i), longitudes(i), & + solar_declination, dt_avg, uniform_angle=rad_uniform_angle) + end do + else + do i = 1, size(latitudes) + ! if dt_avg /= 0, it triggers using avg coszrs + coszrs_rad(i) = shr_orb_cosz(calendar_day, latitudes(i), longitudes(i), & + solar_declination, dt_avg) + end do + end if + call mark_as_initialized('cosine_of_solar_zenith_angle_for_radiation') end if end subroutine orbital_data_advance diff --git a/src/physics/utils/orbital_data.meta b/src/physics/utils/orbital_data.meta index db74793e..bd134e5a 100644 --- a/src/physics/utils/orbital_data.meta +++ b/src/physics/utils/orbital_data.meta @@ -22,9 +22,3 @@ type = real | kind = kind_phys dimensions = (horizontal_dimension) protected = True -[coszrs_rad] - standard_name = cosine_of_solar_zenith_angle_for_radiation - units = rad - type = real | kind = kind_phys - dimensions = (horizontal_dimension) - protected = True From e5384b17a9a751152c2331f8b586c95255210193 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Mon, 13 Oct 2025 00:25:16 -0600 Subject: [PATCH 089/150] fix merge; add pbuf out fields to registry to be checked --- src/data/registry.xml | 32 ++++++++++++++++++++++++++++++++ src/physics/utils/pio_reader.F90 | 1 - 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/data/registry.xml b/src/data/registry.xml index c26192b1..f83e34ba 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -701,6 +701,38 @@ 0.0_kind_phys cflx cam_in_cflx + + horizontal_dimension + 0.0_kind_phys + FSNT pbuf_FSNT + + + horizontal_dimension + 0.0_kind_phys + FSNS pbuf_FSNS + + + horizontal_dimension + 0.0_kind_phys + FLNT pbuf_FLNT + + + horizontal_dimension + 0.0_kind_phys + FLNS pbuf_FLNS + diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 0c1a70bf..23248230 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -2600,7 +2600,6 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims !Check if count indices are within bounds: do i = 1, file_var_dim_num - if ((count(i) < 1) .or. (count(i) > dim_sizes(i))) then if ((count(i) < 1) .or. ((start(i)+count(i)-1) > dim_sizes(i))) then errcode = bad_subset_range_err write(errmsg, '(a,i0,3a,i0,a,i0,a)') & From 7e294571bbca78d206addd5d6bf1d0b0ea84c54e Mon Sep 17 00:00:00 2001 From: peverwhee Date: Mon, 13 Oct 2025 00:25:53 -0600 Subject: [PATCH 090/150] update atmos phys hash --- .gitmodules | 2 +- src/physics/ncar_ccpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9a023928..f9558978 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = aca8edf7c6c6d20902f717d4af5c6d85cbbd81aa + fxtag = 16c146f fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index aca8edf7..16c146fe 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit aca8edf7c6c6d20902f717d4af5c6d85cbbd81aa +Subproject commit 16c146fe5167f1224a8cd88eb3b77ec9f8033291 From b1993e90b3cdf372c4b8bf20849ee6cd9176e5d4 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Mon, 13 Oct 2025 01:44:53 -0600 Subject: [PATCH 091/150] fix runtime object setter; add rrtmgp snapshot test --- cime_config/testdefs/testlist_cam.xml | 9 +++++++ .../cam/outfrq_rrtmgp_derecho/shell_commands | 1 + .../cam/outfrq_rrtmgp_derecho/user_nl_cam | 26 +++++++++++++++++++ .../cam/outfrq_rrtmgp_derecho/user_nl_cpl | 5 ++++ src/control/cam_comp.F90 | 2 +- src/control/runtime_obj.F90 | 9 +++---- src/dynamics/mpas/dyn_comp.F90 | 3 ++- src/dynamics/mpas/dyn_comp_impl.F90 | 7 ++--- src/dynamics/none/dyn_comp.F90 | 7 ++--- src/dynamics/se/dyn_comp.F90 | 5 ++-- 10 files changed, 59 insertions(+), 15 deletions(-) create mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/shell_commands create mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/user_nl_cam create mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/user_nl_cpl diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index da0b7dad..13c077d0 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -88,6 +88,15 @@ + + + + + + + + + diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/shell_commands b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/shell_commands new file mode 100644 index 00000000..9d29c453 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/shell_commands @@ -0,0 +1 @@ + ./xmlchange CAM_CONFIG_OPTS="--dyn none --physics-suites rrtmgp" diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/user_nl_cam new file mode 100644 index 00000000..e862a983 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/user_nl_cam @@ -0,0 +1,26 @@ +! these are FHISTC_LTso snapshots +ncdata = '/glade/campaign/cesm/community/amwg/sima_baselines/cam_sima_test_snapshots/cam_ne3pg3_fhistc_ltso_rrtmgp_derecho_gnu_before_c20251013.nc' +ncdata_check = '/glade/campaign/cesm/community/amwg/sima_baselines/cam_sima_test_snapshots/cam_ne3pg3_fhistc_ltso_rrtmgp_derecho_gnu_after_c20251013.nc' + +! tolerances for testing +ncdata_check_err = .true. +min_difference = 1e-15 + +! vertical levels in snapshot +pver = 58 + +! Do radiation on every timestep we're testing +irad_always=3 + +! diagnostic output +hist_output_frequency;h1: 1*nsteps +hist_precision;h1: REAL64 +hist_add_inst_fields;h1: HR +! Cloud output +hist_add_inst_fields;h1: TOT_CLD_VISTAU,TOT_ICLD_VISTAU,ICE_ICLD_VISTAU,LIQ_ICLD_VISTAU +! Longwave diagnostic output +hist_add_inst_fields;h1: QRL,QRLC,FLNT,FLNTC,FLUT,FLUTC,LWCF,FLN200,FLN200C,FLNR,FLNS,FLNSC,FLDS,FLDSC,FUL,FDL,FULC,FDLC +! Shortwave diagnostic fields +hist_add_inst_fields;h1: SOLIN,QRS,QRSC,FSNT,FSNTC,FSNTOA,FSNTOAC,SWCF,FSUTOA,FSN200,FSN200C,FSNR,SOLL,SOLS,SOLLD,SOLSD +hist_add_inst_fields;h1: FSNS,FSNSC,FSDS,FSDSC,FSNIRTOA,FSNRTOAC,FSNRTOAS,FUS,FDS,FUSC,FDSC +hist_precision;h1: REAL64 diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/user_nl_cpl b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/user_nl_cpl new file mode 100644 index 00000000..08311045 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/user_nl_cpl @@ -0,0 +1,5 @@ +! Set fixed orbital parameters +orb_mode='fixed_parameters' +orb_eccen = 0. +orb_obliq = 0. +orb_mvelp = 0. diff --git a/src/control/cam_comp.F90 b/src/control/cam_comp.F90 index 9df8a387..c94a13a3 100644 --- a/src/control/cam_comp.F90 +++ b/src/control/cam_comp.F90 @@ -236,7 +236,7 @@ subroutine cam_init(caseid, ctitle, model_doi_url, & if (initial_run_in) then - call dyn_init(dyn_in, dyn_out) + call dyn_init(cam_runtime_opts, dyn_in, dyn_out) else diff --git a/src/control/runtime_obj.F90 b/src/control/runtime_obj.F90 index b5c388f1..f77bafa9 100644 --- a/src/control/runtime_obj.F90 +++ b/src/control/runtime_obj.F90 @@ -42,13 +42,13 @@ module runtime_obj procedure, public :: waccmx_option procedure, public :: gw_front procedure, public :: gw_front_igw - procedure, public :: set_dycore procedure, public :: update_thermodynamic_variables end type runtime_options type(runtime_options), public, protected :: cam_runtime_opts public :: cam_set_runtime_opts + public :: set_cam_dycore ! Private data logical :: runtime_configured = .false. @@ -109,13 +109,12 @@ pure logical function update_thermodynamic_variables(self) end function update_thermodynamic_variables - subroutine set_dycore(self, dycore_in) - class(runtime_options), intent(inout) :: self + subroutine set_cam_dycore(dycore_in) character(len=*), intent(in) :: dycore_in - self%dycore = trim(dycore_in) + cam_runtime_opts%dycore = trim(dycore_in) - end subroutine set_dycore + end subroutine set_cam_dycore subroutine cam_set_runtime_opts(phys_suite, waccmx_opt, & gw_front, gw_front_igw) diff --git a/src/dynamics/mpas/dyn_comp.F90 b/src/dynamics/mpas/dyn_comp.F90 index bb4f9bc1..d5f1c9d9 100644 --- a/src/dynamics/mpas/dyn_comp.F90 +++ b/src/dynamics/mpas/dyn_comp.F90 @@ -44,9 +44,10 @@ module subroutine dyn_readnl(namelist_path) character(*), intent(in) :: namelist_path end subroutine dyn_readnl - module subroutine dyn_init(dyn_in, dyn_out) + module subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) use runtime_obj, only: runtime_options + type(runtime_options), intent(in) :: cam_runtime_opts type(dyn_import_t), intent(in) :: dyn_in type(dyn_export_t), intent(in) :: dyn_out end subroutine dyn_init diff --git a/src/dynamics/mpas/dyn_comp_impl.F90 b/src/dynamics/mpas/dyn_comp_impl.F90 index 69602352..46d7931a 100644 --- a/src/dynamics/mpas/dyn_comp_impl.F90 +++ b/src/dynamics/mpas/dyn_comp_impl.F90 @@ -125,7 +125,7 @@ end subroutine dyn_readnl !> (KCW, 2024-05-28) ! ! Called by `cam_init` in `src/control/cam_comp.F90`. - module subroutine dyn_init(dyn_in, dyn_out) + module subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) ! Module(s) from CAM-SIMA. use cam_abortutils, only: check_allocate use cam_constituents, only: const_name, const_is_water_species, num_advected, readtrace @@ -135,13 +135,14 @@ module subroutine dyn_init(dyn_in, dyn_out) use cam_pio_utils, only: clean_iodesc_list use dyn_coupling, only: dyn_exchange_constituent_states use inic_analytic, only: analytic_ic_active - use runtime_obj, only: runtime_options, cam_runtime_opts + use runtime_obj, only: runtime_options, set_cam_dycore use time_manager, only: get_step_size ! Module(s) from CCPP. use phys_vars_init_check, only: std_name_len ! Module(s) from external libraries. use pio, only: file_desc_t + type(runtime_options), intent(in) :: cam_runtime_opts type(dyn_import_t), intent(in) :: dyn_in type(dyn_export_t), intent(in) :: dyn_out @@ -155,7 +156,7 @@ module subroutine dyn_init(dyn_in, dyn_out) type(file_desc_t), pointer :: pio_topo_file ! Set dycore name in runtime object - call cam_runtime_opts%set_dycore('mpas') + call set_cam_dycore('mpas') call dyn_debug_print(debugout_debug, subname // ' entered') diff --git a/src/dynamics/none/dyn_comp.F90 b/src/dynamics/none/dyn_comp.F90 index 2844512e..b0d48508 100644 --- a/src/dynamics/none/dyn_comp.F90 +++ b/src/dynamics/none/dyn_comp.F90 @@ -51,17 +51,18 @@ end subroutine dyn_register !============================================================================== - subroutine dyn_init(dyn_in, dyn_out) - use runtime_obj, only: runtime_options, cam_runtime_opts + subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) + use runtime_obj, only: set_cam_dycore, runtime_options ! Null dycore, no action ! Dummy arguments: + type(runtime_options), intent(in) :: cam_runtime_opts type(dyn_import_t), intent(out) :: dyn_in type(dyn_export_t), intent(out) :: dyn_out ! Note: dynamical core energy formula is set in dyn_grid based on dynamical core ! that provided the initial conditions file - call cam_runtime_opts%set_dycore('null') + call set_cam_dycore('null') end subroutine dyn_init diff --git a/src/dynamics/se/dyn_comp.F90 b/src/dynamics/se/dyn_comp.F90 index 8fac2ad5..a35188e9 100644 --- a/src/dynamics/se/dyn_comp.F90 +++ b/src/dynamics/se/dyn_comp.F90 @@ -557,7 +557,7 @@ end subroutine dyn_readnl !========================================================================================= subroutine dyn_init(dyn_in, dyn_out) - use runtime_obj, only: runtime_options, cam_runtime_opts + use runtime_obj, only: runtime_options, set_cam_dycore use dyn_grid, only: elem, fvm use cam_pio_utils, only: clean_iodesc_list use air_composition, only: thermodynamic_active_species_num, thermodynamic_active_species_idx @@ -586,6 +586,7 @@ subroutine dyn_init(dyn_in, dyn_out) use control_mod, only: vert_remap_uvTq_alg, vert_remap_tracer_alg ! Dummy arguments: + type(runtime_options), intent(in) :: cam_runtime_opts type(dyn_import_t), intent(out) :: dyn_in type(dyn_export_t), intent(out) :: dyn_out @@ -654,7 +655,7 @@ subroutine dyn_init(dyn_in, dyn_out) call mark_as_initialized("flag_for_dycore_energy_consistency_adjustment") ! Set name of dycore in runtime object - call cam_runtime_opts%set_dycore('se') + call set_cam_dycore('se') ! Now allocate and set condenstate vars allocate(cnst_name_gll(qsize), stat=iret) ! constituent names for gll tracers From 6c0492493dcb3f2dd34400bd26f6b98562d2a4e7 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Mon, 13 Oct 2025 11:33:35 -0400 Subject: [PATCH 092/150] Update do_molec_diff standard name to do_molecular_diffusion --- src/data/ref_pres.F90 | 2 +- src/data/ref_pres.meta | 2 +- test/unit/python/sample_files/ref_pres.meta | 2 +- test/unit/python/sample_files/ref_pres_SourceMods.meta | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/data/ref_pres.F90 b/src/data/ref_pres.F90 index bf4bda45..98182467 100644 --- a/src/data/ref_pres.F90 +++ b/src/data/ref_pres.F90 @@ -185,7 +185,7 @@ subroutine ref_pres_init(pref_edge_in, pref_mid_in, num_pr_lev_in) ! molec_diff_bot_press call mark_as_initialized("pressure_at_bottom_of_molecular_diffusion") ! do_molec_diff - call mark_as_initialized("flag_for_molecular_diffusion") + call mark_as_initialized("do_molecular_diffusion") ! nbot_molec call mark_as_initialized("index_of_pressure_at_bottom_of_molecular_diffusion") diff --git a/src/data/ref_pres.meta b/src/data/ref_pres.meta index 79f9f319..70486b7f 100644 --- a/src/data/ref_pres.meta +++ b/src/data/ref_pres.meta @@ -59,7 +59,7 @@ dimensions = () protected = True [ do_molec_diff ] - standard_name = flag_for_molecular_diffusion + standard_name = do_molecular_diffusion units = flag type = logical dimensions = () diff --git a/test/unit/python/sample_files/ref_pres.meta b/test/unit/python/sample_files/ref_pres.meta index 4afd23c3..ba4140bd 100644 --- a/test/unit/python/sample_files/ref_pres.meta +++ b/test/unit/python/sample_files/ref_pres.meta @@ -65,7 +65,7 @@ type = real | kind = kind_phys dimensions = () [ do_molec_diff ] - standard_name = flag_for_molecular_diffusion + standard_name = do_molecular_diffusion units = flag type = logical dimensions = () diff --git a/test/unit/python/sample_files/ref_pres_SourceMods.meta b/test/unit/python/sample_files/ref_pres_SourceMods.meta index 7a42dc99..141eb2fd 100644 --- a/test/unit/python/sample_files/ref_pres_SourceMods.meta +++ b/test/unit/python/sample_files/ref_pres_SourceMods.meta @@ -65,7 +65,7 @@ type = real | kind = kind_phys dimensions = () [ do_molec_diff ] - standard_name = flag_for_molecular_diffusion + standard_name = do_molecular_diffusion units = flag type = logical dimensions = () From d47bb6f51fdf424c0e512817716ee58180b69078 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Mon, 13 Oct 2025 14:55:40 -0400 Subject: [PATCH 093/150] Remove TBD from GW names --- src/data/registry.xml | 2 +- .../utils/gravity_wave_drag_ridge_read.F90 | 30 +++++------ .../utils/gravity_wave_drag_ridge_read.meta | 50 +++++++++---------- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/data/registry.xml b/src/data/registry.xml index d2eb00aa..39ef7354 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -1370,7 +1370,7 @@ horizontal_dimension vertical_layer_dimension diff --git a/src/physics/utils/gravity_wave_drag_ridge_read.F90 b/src/physics/utils/gravity_wave_drag_ridge_read.F90 index f22d08ab..b329d89e 100644 --- a/src/physics/utils/gravity_wave_drag_ridge_read.F90 +++ b/src/physics/utils/gravity_wave_drag_ridge_read.F90 @@ -145,7 +145,7 @@ subroutine gravity_wave_drag_ridge_read_file() errflg = 0 has_gbxar_from_topo = .false. - call mark_as_initialized('number_of_ridges_in_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('number_of_ridges_in_ridge_gravity_wave_drag') ! Do we have meso-Beta file? if(bnd_topo /= unset_path_str) then @@ -250,14 +250,14 @@ subroutine gravity_wave_drag_ridge_read_file() endif ! Mark variables as initialized so they are not read from ic file. - call mark_as_initialized('grid_box_area_for_beta_ridge_gravity_wave_drag_tbd') - call mark_as_initialized('isotropic_variance_for_beta_ridge_gravity_wave_drag_tbd') - call mark_as_initialized('isotropic_weight_for_beta_ridge_gravity_wave_drag_tbd') - call mark_as_initialized('ridge_half_width_for_beta_ridge_gravity_wave_drag_tbd') - call mark_as_initialized('ridge_length_for_beta_ridge_gravity_wave_drag_tbd') - call mark_as_initialized('ridge_obstacle_height_for_beta_ridge_gravity_wave_drag_tbd') - call mark_as_initialized('ridge_anisotropy_for_beta_ridge_gravity_wave_drag_tbd') - call mark_as_initialized('ridge_clockwise_angle_from_north_for_beta_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('grid_box_area_for_beta_ridge_gravity_wave_drag') + call mark_as_initialized('isotropic_variance_for_beta_ridge_gravity_wave_drag') + call mark_as_initialized('isotropic_weight_for_beta_ridge_gravity_wave_drag') + call mark_as_initialized('ridge_half_width_for_beta_ridge_gravity_wave_drag') + call mark_as_initialized('ridge_length_for_beta_ridge_gravity_wave_drag') + call mark_as_initialized('ridge_obstacle_height_for_beta_ridge_gravity_wave_drag') + call mark_as_initialized('ridge_anisotropy_for_beta_ridge_gravity_wave_drag') + call mark_as_initialized('ridge_clockwise_angle_from_north_for_beta_ridge_gravity_wave_drag') endif ! Do we have meso-Gamma file? @@ -348,12 +348,12 @@ subroutine gravity_wave_drag_ridge_read_file() deallocate(fh_rdggm) ! Mark variables as initialized so they are not read from ic file. - call mark_as_initialized('grid_box_area_for_gamma_ridge_gravity_wave_drag_tbd') - call mark_as_initialized('ridge_half_width_for_gamma_ridge_gravity_wave_drag_tbd') - call mark_as_initialized('ridge_length_for_gamma_ridge_gravity_wave_drag_tbd') - call mark_as_initialized('ridge_obstacle_height_for_gamma_ridge_gravity_wave_drag_tbd') - call mark_as_initialized('ridge_anisotropy_for_gamma_ridge_gravity_wave_drag_tbd') - call mark_as_initialized('ridge_clockwise_angle_from_north_for_gamma_ridge_gravity_wave_drag_tbd') + call mark_as_initialized('grid_box_area_for_gamma_ridge_gravity_wave_drag') + call mark_as_initialized('ridge_half_width_for_gamma_ridge_gravity_wave_drag') + call mark_as_initialized('ridge_length_for_gamma_ridge_gravity_wave_drag') + call mark_as_initialized('ridge_obstacle_height_for_gamma_ridge_gravity_wave_drag') + call mark_as_initialized('ridge_anisotropy_for_gamma_ridge_gravity_wave_drag') + call mark_as_initialized('ridge_clockwise_angle_from_north_for_gamma_ridge_gravity_wave_drag') endif end subroutine gravity_wave_drag_ridge_read_file diff --git a/src/physics/utils/gravity_wave_drag_ridge_read.meta b/src/physics/utils/gravity_wave_drag_ridge_read.meta index 2d62af8c..e3dd480b 100644 --- a/src/physics/utils/gravity_wave_drag_ridge_read.meta +++ b/src/physics/utils/gravity_wave_drag_ridge_read.meta @@ -6,77 +6,77 @@ name = gravity_wave_drag_ridge_read type = module [ prdg ] - standard_name = number_of_ridges_in_ridge_gravity_wave_drag_tbd + standard_name = number_of_ridges_in_ridge_gravity_wave_drag units = count type = integer dimensions = () [ rdg_gbxar ] - standard_name = grid_box_area_for_beta_ridge_gravity_wave_drag_tbd + standard_name = grid_box_area_for_beta_ridge_gravity_wave_drag units = km2 type = real | kind = kind_phys dimensions = (horizontal_dimension) [ rdg_isovar ] - standard_name = isotropic_variance_for_beta_ridge_gravity_wave_drag_tbd + standard_name = isotropic_variance_for_beta_ridge_gravity_wave_drag units = m type = real | kind = kind_phys dimensions = (horizontal_dimension) [ rdg_isowgt ] - standard_name = isotropic_weight_for_beta_ridge_gravity_wave_drag_tbd + standard_name = isotropic_weight_for_beta_ridge_gravity_wave_drag units = 1 type = real | kind = kind_phys dimensions = (horizontal_dimension) [ rdg_hwdth ] - standard_name = ridge_half_width_for_beta_ridge_gravity_wave_drag_tbd + standard_name = ridge_half_width_for_beta_ridge_gravity_wave_drag units = km type = real | kind = kind_phys - dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag) [ rdg_clngt ] - standard_name = ridge_length_for_beta_ridge_gravity_wave_drag_tbd + standard_name = ridge_length_for_beta_ridge_gravity_wave_drag units = km type = real | kind = kind_phys - dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag) [ rdg_mxdis ] - standard_name = ridge_obstacle_height_for_beta_ridge_gravity_wave_drag_tbd + standard_name = ridge_obstacle_height_for_beta_ridge_gravity_wave_drag units = m type = real | kind = kind_phys - dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag) [ rdg_anixy ] - standard_name = ridge_anisotropy_for_beta_ridge_gravity_wave_drag_tbd + standard_name = ridge_anisotropy_for_beta_ridge_gravity_wave_drag units = 1 type = real | kind = kind_phys - dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag) [ rdg_angll ] - standard_name = ridge_clockwise_angle_from_north_for_beta_ridge_gravity_wave_drag_tbd + standard_name = ridge_clockwise_angle_from_north_for_beta_ridge_gravity_wave_drag units = degree type = real | kind = kind_phys - dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag) [ rdg_gbxarg ] - standard_name = grid_box_area_for_gamma_ridge_gravity_wave_drag_tbd + standard_name = grid_box_area_for_gamma_ridge_gravity_wave_drag units = km2 type = real | kind = kind_phys dimensions = (horizontal_dimension) [ rdg_hwdthg ] - standard_name = ridge_half_width_for_gamma_ridge_gravity_wave_drag_tbd + standard_name = ridge_half_width_for_gamma_ridge_gravity_wave_drag units = km type = real | kind = kind_phys - dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag) [ rdg_clngtg ] - standard_name = ridge_length_for_gamma_ridge_gravity_wave_drag_tbd + standard_name = ridge_length_for_gamma_ridge_gravity_wave_drag units = km type = real | kind = kind_phys - dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag) [ rdg_mxdisg ] - standard_name = ridge_obstacle_height_for_gamma_ridge_gravity_wave_drag_tbd + standard_name = ridge_obstacle_height_for_gamma_ridge_gravity_wave_drag units = m type = real | kind = kind_phys - dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag) [ rdg_anixyg ] - standard_name = ridge_anisotropy_for_gamma_ridge_gravity_wave_drag_tbd + standard_name = ridge_anisotropy_for_gamma_ridge_gravity_wave_drag units = 1 type = real | kind = kind_phys - dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag) [ rdg_angllg ] - standard_name = ridge_clockwise_angle_from_north_for_gamma_ridge_gravity_wave_drag_tbd + standard_name = ridge_clockwise_angle_from_north_for_gamma_ridge_gravity_wave_drag units = degree type = real | kind = kind_phys - dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag_tbd) + dimensions = (horizontal_dimension, number_of_ridges_in_ridge_gravity_wave_drag) From 679e11f5c2f905cac8937efb4a1221fd2eaf1bf9 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Mon, 13 Oct 2025 15:01:58 -0400 Subject: [PATCH 094/150] Fix closing tag in registry.xml due to bad git merge --- src/data/registry.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/data/registry.xml b/src/data/registry.xml index 39ef7354..d279e427 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -1391,6 +1391,7 @@ horizontal_dimension 0.0_kind_phys pbuf_SGH + Date: Mon, 13 Oct 2025 15:54:52 -0400 Subject: [PATCH 095/150] Calculate gw top taper in ref_pres to pass to scheme via metadata --- src/data/ref_pres.F90 | 15 +++++++++++++++ src/data/ref_pres.meta | 11 +++++++++++ 2 files changed, 26 insertions(+) diff --git a/src/data/ref_pres.F90 b/src/data/ref_pres.F90 index 98182467..58c1c3e9 100644 --- a/src/data/ref_pres.F90 +++ b/src/data/ref_pres.F90 @@ -56,6 +56,12 @@ module ref_pres logical, protected :: do_molec_diff = .false. integer, protected :: nbot_molec = 0 + ! Pressure limit used to set gravity wave tapering at top of model (Pa) + real(kind_phys), parameter :: gravity_wave_taper_bot_press = 0.6E-02_kind_phys + + ! Bottom level for tapering gravity waves at top of model + integer, protected :: nbot_gravity_wave_top_taper = 0 + !==================================================================================== contains !==================================================================================== @@ -165,6 +171,11 @@ subroutine ref_pres_init(pref_edge_in, pref_mid_in, num_pr_lev_in) top=.false.) end if + ! Find level corresponding to the bottom for tapering gravity waves + ! at the top of model. + nbot_gravity_wave_top_taper = press_lim_idx(gravity_wave_taper_bot_press, & + top=.true.) + ! Tell rest of model that variables have been initialized: ! pref_edge_in call mark_as_initialized("reference_pressure_at_interface") @@ -188,6 +199,10 @@ subroutine ref_pres_init(pref_edge_in, pref_mid_in, num_pr_lev_in) call mark_as_initialized("do_molecular_diffusion") ! nbot_molec call mark_as_initialized("index_of_pressure_at_bottom_of_molecular_diffusion") + ! gravity_wave_taper_bot_press + call mark_as_initialized("largest_model_top_pressure_that_allows_tapering_gravity_wave_drag_at_model_top") + ! nbot_gravity_wave_top_taper + call mark_as_initialized("vertical_index_of_bottom_limit_for_tapering_gravity_wave_drag_at_model_top") end subroutine ref_pres_init diff --git a/src/data/ref_pres.meta b/src/data/ref_pres.meta index 70486b7f..953c251f 100644 --- a/src/data/ref_pres.meta +++ b/src/data/ref_pres.meta @@ -70,3 +70,14 @@ type = integer dimensions = () protected = True +[ gravity_wave_taper_bot_press ] + standard_name = largest_model_top_pressure_that_allows_tapering_gravity_wave_drag_at_model_top + units = Pa + type = real | kind = kind_phys + dimensions = () +[ nbot_gravity_wave_top_taper ] + standard_name = vertical_index_of_bottom_limit_for_tapering_gravity_wave_drag_at_model_top + units = index + type = integer + dimensions = () + protected = True From fa9acd0d0d283ba8d4895792cdb8edc549f8e249 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Mon, 13 Oct 2025 20:28:46 -0600 Subject: [PATCH 096/150] remove redundant git-fleximod --- .lib/CODE_OF_CONDUCT.md | 107 --- .lib/License | 20 - .lib/README.md | 108 --- .lib/doc/Makefile | 20 - .lib/doc/conf.py | 26 - .lib/doc/index.rst | 24 - .lib/doc/make.bat | 35 - .lib/escomp_install | 25 - .../git-fleximod/.github/workflows/pre-commit | 13 - .../.github/workflows/pytest.yaml | 77 -- .lib/git-fleximod/.pre-commit-config.yaml | 18 - .lib/git-fleximod/CODE_OF_CONDUCT.md | 107 --- .lib/git-fleximod/License | 20 - .lib/git-fleximod/README.md | 108 --- .lib/git-fleximod/doc/Makefile | 20 - .lib/git-fleximod/doc/conf.py | 26 - .lib/git-fleximod/doc/index.rst | 24 - .lib/git-fleximod/doc/make.bat | 35 - .lib/git-fleximod/escomp_install | 25 - .lib/git-fleximod/git_fleximod/__init__.py | 0 .lib/git-fleximod/git_fleximod/cli.py | 145 ---- .../git-fleximod/git_fleximod/git_fleximod.py | 370 ---------- .../git-fleximod/git_fleximod/gitinterface.py | 126 ---- .lib/git-fleximod/git_fleximod/gitmodules.py | 97 --- .../git-fleximod/git_fleximod/lstripreader.py | 43 -- .lib/git-fleximod/git_fleximod/metoflexi.py | 236 ------ .lib/git-fleximod/git_fleximod/submodule.py | 459 ------------ .lib/git-fleximod/git_fleximod/utils.py | 365 --------- .lib/git-fleximod/poetry.lock | 693 ------------------ .lib/git-fleximod/pyproject.toml | 42 -- .lib/git-fleximod/tbump.toml | 43 -- .lib/git-fleximod/tests/__init__.py | 3 - .lib/git-fleximod/tests/conftest.py | 150 ---- .lib/git-fleximod/tests/test_a_import.py | 8 - .lib/git-fleximod/tests/test_b_update.py | 26 - .lib/git-fleximod/tests/test_c_required.py | 43 -- .lib/git-fleximod/tests/test_d_complex.py | 66 -- .../tests/test_e_complex_update.py | 69 -- .lib/poetry.lock | 693 ------------------ .lib/pyproject.toml | 42 -- .lib/tbump.toml | 43 -- .lib/tests/__init__.py | 3 - .lib/tests/conftest.py | 150 ---- .lib/tests/test_a_import.py | 8 - .lib/tests/test_b_update.py | 26 - .lib/tests/test_c_required.py | 30 - .lib/tests/test_d_complex.py | 66 -- .lib/tests/test_e_complex_update.py | 69 -- 48 files changed, 4952 deletions(-) delete mode 100644 .lib/CODE_OF_CONDUCT.md delete mode 100644 .lib/License delete mode 100644 .lib/README.md delete mode 100644 .lib/doc/Makefile delete mode 100644 .lib/doc/conf.py delete mode 100644 .lib/doc/index.rst delete mode 100644 .lib/doc/make.bat delete mode 100644 .lib/escomp_install delete mode 100644 .lib/git-fleximod/.github/workflows/pre-commit delete mode 100644 .lib/git-fleximod/.github/workflows/pytest.yaml delete mode 100644 .lib/git-fleximod/.pre-commit-config.yaml delete mode 100644 .lib/git-fleximod/CODE_OF_CONDUCT.md delete mode 100644 .lib/git-fleximod/License delete mode 100644 .lib/git-fleximod/README.md delete mode 100644 .lib/git-fleximod/doc/Makefile delete mode 100644 .lib/git-fleximod/doc/conf.py delete mode 100644 .lib/git-fleximod/doc/index.rst delete mode 100644 .lib/git-fleximod/doc/make.bat delete mode 100644 .lib/git-fleximod/escomp_install delete mode 100644 .lib/git-fleximod/git_fleximod/__init__.py delete mode 100644 .lib/git-fleximod/git_fleximod/cli.py delete mode 100755 .lib/git-fleximod/git_fleximod/git_fleximod.py delete mode 100644 .lib/git-fleximod/git_fleximod/gitinterface.py delete mode 100644 .lib/git-fleximod/git_fleximod/gitmodules.py delete mode 100644 .lib/git-fleximod/git_fleximod/lstripreader.py delete mode 100755 .lib/git-fleximod/git_fleximod/metoflexi.py delete mode 100644 .lib/git-fleximod/git_fleximod/submodule.py delete mode 100644 .lib/git-fleximod/git_fleximod/utils.py delete mode 100644 .lib/git-fleximod/poetry.lock delete mode 100644 .lib/git-fleximod/pyproject.toml delete mode 100644 .lib/git-fleximod/tbump.toml delete mode 100644 .lib/git-fleximod/tests/__init__.py delete mode 100644 .lib/git-fleximod/tests/conftest.py delete mode 100644 .lib/git-fleximod/tests/test_a_import.py delete mode 100644 .lib/git-fleximod/tests/test_b_update.py delete mode 100644 .lib/git-fleximod/tests/test_c_required.py delete mode 100644 .lib/git-fleximod/tests/test_d_complex.py delete mode 100644 .lib/git-fleximod/tests/test_e_complex_update.py delete mode 100644 .lib/poetry.lock delete mode 100644 .lib/pyproject.toml delete mode 100644 .lib/tbump.toml delete mode 100644 .lib/tests/__init__.py delete mode 100644 .lib/tests/conftest.py delete mode 100644 .lib/tests/test_a_import.py delete mode 100644 .lib/tests/test_b_update.py delete mode 100644 .lib/tests/test_c_required.py delete mode 100644 .lib/tests/test_d_complex.py delete mode 100644 .lib/tests/test_e_complex_update.py diff --git a/.lib/CODE_OF_CONDUCT.md b/.lib/CODE_OF_CONDUCT.md deleted file mode 100644 index 84f2925b..00000000 --- a/.lib/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,107 +0,0 @@ -# Contributor Code of Conduct -_The Contributor Code of Conduct is for participants in our software projects and community._ - -## Our Pledge -We, as contributors, creators, stewards, and maintainers (participants), of **git-fleximod** pledge to make participation in -our software, system or hardware project and community a safe, productive, welcoming and inclusive experience for everyone. -All participants are required to abide by this Code of Conduct. -This includes respectful treatment of everyone regardless of age, body size, disability, ethnicity, gender identity or expression, -level of experience, nationality, political affiliation, veteran status, pregnancy, genetic information, physical appearance, race, -religion, or sexual orientation, as well as any other characteristic protected under applicable US federal or state law. - -## Our Standards -Examples of behaviors that contribute to a positive environment include: - -* All participants are treated with respect and consideration, valuing a diversity of views and opinions -* Be considerate, respectful, and collaborative -* Communicate openly with respect for others, critiquing ideas rather than individuals and gracefully accepting criticism -* Acknowledging the contributions of others -* Avoid personal attacks directed toward other participants -* Be mindful of your surroundings and of your fellow participants -* Alert UCAR staff and suppliers/vendors if you notice a dangerous situation or someone in distress -* Respect the rules and policies of the project and venue - -Examples of unacceptable behavior include, but are not limited to: - -* Harassment, intimidation, or discrimination in any form -* Physical, verbal, or written abuse by anyone to anyone, including repeated use of pronouns other than those requested -* Unwelcome sexual attention or advances -* Personal attacks directed at other guests, members, participants, etc. -* Publishing others' private information, such as a physical or electronic address, without explicit permission -* Alarming, intimidating, threatening, or hostile comments or conduct -* Inappropriate use of nudity and/or sexual images -* Threatening or stalking anyone, including a participant -* Other conduct which could reasonably be considered inappropriate in a professional setting - -## Scope -This Code of Conduct applies to all spaces managed by the Project whether they be physical, online or face-to-face. -This includes project code, code repository, associated web pages, documentation, mailing lists, project websites and wiki pages, -issue tracker, meetings, telecons, events, project social media accounts, and any other forums created by the project team which the -community uses for communication. -In addition, violations of this Code of Conduct outside these spaces may affect a person's ability to participate within them. -Representation of a project may be further defined and clarified by project maintainers. - -## Community Responsibilities -Everyone in the community is empowered to respond to people who are showing unacceptable behavior. -They can talk to them privately or publicly. -Anyone requested to stop unacceptable behavior is expected to comply immediately. -If the behavior continues concerns may be brought to the project administrators or to any other party listed in the -[Reporting](#reporting) section below. - -## Project Administrator Responsibilities -Project administrators are responsible for clarifying the standards of acceptable behavior and are encouraged to model appropriate -behavior and provide support when people in the community point out inappropriate behavior. -Project administrator(s) are normally the ones that would be tasked to carry out the actions in the [Consequences](#consequences) -section below. - -Project administrators are also expected to keep this Code of Conduct updated with the main one housed at UCAR, as listed below in -the [Attribution](#attribution) section. - -## Reporting -Instances of unacceptable behavior can be brought to the attention of the project administrator(s) who may take any action as -outlined in the [Consequences](#consequences) section below. -However, making a report to a project administrator is not considered an 'official report' to UCAR. - -Instances of unacceptable behavior may also be reported directly to UCAR pursuant to [UCAR's Harassment Reporting and Complaint -Procedure](https://www2.fin.ucar.edu/procedures/hr/harassment-reporting-and-complaint-procedure), or anonymously through [UCAR's -EthicsPoint Hotline](https://www2.fin.ucar.edu/ethics/anonymous-reporting). - -Complaints received by UCAR will be handled pursuant to the procedures outlined in UCAR's Harassment Reporting and Complaint -Procedure. -Complaints to UCAR will be held as confidential as practicable under the circumstances, and retaliation against a person who -initiates a complaint or an inquiry about inappropriate behavior will not be tolerated. - -Any Contributor can use these reporting methods even if they are not directly affiliated with UCAR. -The Frequently Asked Questions (FAQ) page for reporting is [here](https://www2.fin.ucar.edu/procedures/hr/reporting-faqs). - -## Consequences -Upon receipt of a complaint, the project administrator(s) may take any action deemed necessary and appropriate under the -circumstances. -Such action can include things such as: removing, editing, or rejecting comments, commits, code, wiki edits, email, issues, and -other contributions that are not aligned to this Code of Conduct, or banning temporarily or permanently any contributor for other -behaviors that are deemed inappropriate, threatening, offensive, or harmful. -Project administrators also have the right to report violations to UCAR HR and/or UCAR's Office of Diversity, Equity and Inclusion -(ODEI), as well as a participant's home institution and/or law enforcement. -In the event an incident is reported to UCAR, UCAR will follow its Harassment Reporting and Complaint Procedure. - -## Process for Changes -All UCAR managed projects are required to adopt this Contributor Code of Conduct. -Adoption is assumed even if not expressly stated in the repository. -Projects should fill in sections where prompted with project-specific information, including, project name and adoption date. - -Projects that adopt this Code of Conduct need to stay up to date with UCAR's Contributor Code of Conduct, linked with a DOI in the -[Attribution](#attribution) section below. -Projects can make limited substantive changes to the Code of Conduct, however, the changes must be limited in scope and may not -contradict the UCAR Contributor Code of Conduct. - -## Attribution -This Code of Conduct was originally adapted from the [Contributor Covenant](http://contributor-covenant.org/version/1/4), version -1.4. -We then aligned it with the UCAR Participant Code of Conduct, which also borrows from the American Geophysical Union (AGU) Code of -Conduct. -The UCAR Participant Code of Conduct applies to both UCAR employees as well as participants in activities run by UCAR. -The original version of this for all software projects that have strong management from UCAR or UCAR staff is available on the UCAR -website at https://doi.org/10.5065/6w2c-a132. -The date that it was adopted by this project was **Feb/13/2018**. -When responding to complaints, UCAR HR and ODEI will do so based on the latest published version. -Therefore, any project-specific changes should follow the [Process for Changes](#process-for-changes) section above. diff --git a/.lib/License b/.lib/License deleted file mode 100644 index 88bc2251..00000000 --- a/.lib/License +++ /dev/null @@ -1,20 +0,0 @@ -Copyright 2024 NSF National Center for Atmospheric Sciences (NCAR) - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -“Software”), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/.lib/README.md b/.lib/README.md deleted file mode 100644 index 53917da4..00000000 --- a/.lib/README.md +++ /dev/null @@ -1,108 +0,0 @@ -# git-fleximod - -Flexible, Enhanced Submodule Management for Git - -## Overview - -Git-fleximod is a Python-based tool that extends Git's submodule and sparse checkout capabilities, offering additional features for managing submodules in a more flexible and efficient way. - -## Installation - - If you choose to locate git-fleximod in your path you can access it via command: git fleximod - -## Usage - - Basic Usage: - git fleximod [options] - Available Commands: - status: Display the status of submodules. - update: Update submodules to the tag indicated in .gitmodules variable fxtag. - test: Make sure that fxtags and submodule hashes are consistant, - make sure that official urls (as defined by fxDONOTUSEurl) are set - make sure that fxtags are defined for all submodules - Additional Options: - See git fleximod --help for more details. - -## Supported .gitmodules Variables - - fxtag: Specify a specific tag or branch to checkout for a submodule. - fxrequired: Mark a submodule's checkout behavior, with allowed values: - - ToplevelRequired: Top-level and required (checked out only when this is the Toplevel module). - - ToplevelOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module). - - AlwaysRequired: Always required (always checked out). - - AlwaysOptional: Always optional (checked out with --optional flag). - fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths. - fxDONOTUSEurl: This is the url used in the test subcommand to assure that protected branches do not point to forks - **NOTE** the fxDONOTUSEurl variable is only used to identify the official project repository and should not be - changed by users. Use the url variable to change to a fork if desired. - -## Sparse Checkouts - - To enable sparse checkout for a submodule, set the fxsparse variable - in the .gitmodules file to the path of a file containing the desired - sparse checkout paths. Git-fleximod will automatically configure - sparse checkout based on this file when applicable commands are run. - See [git-sparse-checkout](https://git-scm.com/docs/git-sparse-checkout#_internalsfull_pattern_set) - for details on the format of this file. - -## Tests - - The git fleximod test action is designed to be used by, for example, github workflows - to assure that protected branches are consistant with respect to submodule hashes and fleximod fxtags - -## Examples - -Here are some common usage examples: - -Update all submodules, including optional ones: -```bash - git fleximod update --optional -``` - -Updating a specific submodule to the fxtag indicated in .gitmodules: - -```bash - git fleximod update submodule-name -``` -Example .gitmodules entry: -```ini, toml - [submodule "cosp2"] - path = src/physics/cosp2/src - url = https://github.com/CFMIP/COSPv2.0 - fxsparse = ../.cosp_sparse_checkout - fxrequired = AlwaysRequired - fxtag = v2.1.4cesm -``` -Explanation: - -This entry indicates that the submodule named cosp2 at tag v2.1.4cesm -should be checked out into the directory src/physics/cosp2/src -relative to the .gitmodules directory. It should be checked out from -the URL https://github.com/CFMIP/COSPv2.0 and use sparse checkout as -described in the file ../.cosp_sparse_checkout relative to the path -directory. It should be checked out anytime this .gitmodules entry is -read. - -Additional example: -```ini, toml - [submodule "cime"] - path = cime - url = https://github.com/jedwards4b/cime - fxrequired = ToplevelRequired - fxtag = cime6.0.198_rme01 -``` - -Explanation: - -This entry indicates that the submodule cime should be checked out -into a directory named cime at tag cime6.0.198_rme01 from the URL -https://github.com/jedwards4b/cime. This should only be done if -the .gitmodules file is at the top level of the repository clone. - -## Contributing - -We welcome contributions! Please see the CONTRIBUTING.md file for guidelines. - -## License - -Git-fleximod is released under the MIT License. diff --git a/.lib/doc/Makefile b/.lib/doc/Makefile deleted file mode 100644 index d4bb2cbb..00000000 --- a/.lib/doc/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/.lib/doc/conf.py b/.lib/doc/conf.py deleted file mode 100644 index 423099ee..00000000 --- a/.lib/doc/conf.py +++ /dev/null @@ -1,26 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# For the full list of built-in configuration values, see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Project information ----------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information - -project = "git-fleximod" -author = "Jim Edwards " -release = "0.4.0" - -# -- General configuration --------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration - -extensions = ["sphinx_argparse_cli"] - -templates_path = ["_templates"] -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - - -# -- Options for HTML output ------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output - -html_theme = "alabaster" -html_static_path = ["_static"] diff --git a/.lib/doc/index.rst b/.lib/doc/index.rst deleted file mode 100644 index 0f9c1a7f..00000000 --- a/.lib/doc/index.rst +++ /dev/null @@ -1,24 +0,0 @@ -.. git-fleximod documentation master file, created by - sphinx-quickstart on Sat Feb 3 12:02:22 2024. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to git-fleximod's documentation! -======================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: -.. module:: sphinxcontrib.autoprogram -.. sphinx_argparse_cli:: - :module: git_fleximod.cli - :func: get_parser - :prog: git-fleximod - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/.lib/doc/make.bat b/.lib/doc/make.bat deleted file mode 100644 index 32bb2452..00000000 --- a/.lib/doc/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "" goto help - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/.lib/escomp_install b/.lib/escomp_install deleted file mode 100644 index ae782e72..00000000 --- a/.lib/escomp_install +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# updates git-fleximod in an ESCOMP model -# this script should be run from the model root directory, it expects -# git-fleximod to already be installed with the script in bin -# and the classes in lib/python/site-packages -import sys -import shutil -import os - -from glob import iglob - -fleximod_root = sys.argv[1] -fleximod_path = os.path.join(fleximod_root,"src","git-fleximod") -if os.path.isfile(fleximod_path): - with open(fleximod_path,"r") as f: - fleximod = f.readlines() - with open(os.path.join(".","bin","git-fleximod"),"w") as f: - for line in fleximod: - f.write(line) - if "import argparse" in line: - f.write('\nsys.path.append(os.path.join(os.path.dirname(__file__),"..","lib","python","site-packages"))\n\n') - - for file in iglob(os.path.join(fleximod_root, "src", "fleximod", "*.py")): - shutil.copy(file, - os.path.join("lib","python","site-packages","fleximod",os.path.basename(file))) diff --git a/.lib/git-fleximod/.github/workflows/pre-commit b/.lib/git-fleximod/.github/workflows/pre-commit deleted file mode 100644 index 1a6ad008..00000000 --- a/.lib/git-fleximod/.github/workflows/pre-commit +++ /dev/null @@ -1,13 +0,0 @@ -name: pre-commit -on: - pull_request: - push: - branches: [main] - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: pre-commit/action@v3.0.0 diff --git a/.lib/git-fleximod/.github/workflows/pytest.yaml b/.lib/git-fleximod/.github/workflows/pytest.yaml deleted file mode 100644 index 0868dd9a..00000000 --- a/.lib/git-fleximod/.github/workflows/pytest.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# Run this job on pushes to `main`, and for pull requests. If you don't specify -# `branches: [main], then this actions runs _twice_ on pull requests, which is -# annoying. - -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - # If you wanted to use multiple Python versions, you'd have specify a matrix in the job and - # reference the matrixe python version here. - - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - # Cache the installation of Poetry itself, e.g. the next step. This prevents the workflow - # from installing Poetry every time, which can be slow. Note the use of the Poetry version - # number in the cache key, and the "-0" suffix: this allows you to invalidate the cache - # manually if/when you want to upgrade Poetry, or if something goes wrong. This could be - # mildly cleaner by using an environment variable, but I don't really care. - - name: cache poetry install - uses: actions/cache@v4 - with: - path: ~/.local - key: poetry-1.7.1 - - # Install Poetry. You could do this manually, or there are several actions that do this. - # `snok/install-poetry` seems to be minimal yet complete, and really just calls out to - # Poetry's default install script, which feels correct. I pin the Poetry version here - # because Poetry does occasionally change APIs between versions and I don't want my - # actions to break if it does. - # - # The key configuration value here is `virtualenvs-in-project: true`: this creates the - # venv as a `.venv` in your testing directory, which allows the next step to easily - # cache it. - - uses: snok/install-poetry@v1 - with: - version: 1.7.1 - virtualenvs-create: true - virtualenvs-in-project: true - - # Cache your dependencies (i.e. all the stuff in your `pyproject.toml`). Note the cache - # key: if you're using multiple Python versions, or multiple OSes, you'd need to include - # them in the cache key. I'm not, so it can be simple and just depend on the poetry.lock. - - name: cache deps - id: cache-deps - uses: actions/cache@v4 - with: - path: .venv - key: pydeps-${{ hashFiles('**/poetry.lock') }} - - # Install dependencies. `--no-root` means "install all dependencies but not the project - # itself", which is what you want to avoid caching _your_ code. The `if` statement - # ensures this only runs on a cache miss. - - run: poetry install --no-interaction --no-root - if: steps.cache-deps.outputs.cache-hit != 'true' - - # Now install _your_ project. This isn't necessary for many types of projects -- particularly - # things like Django apps don't need this. But it's a good idea since it fully-exercises the - # pyproject.toml and makes that if you add things like console-scripts at some point that - # they'll be installed and working. - - run: poetry install --no-interaction - - # And finally run tests. I'm using pytest and all my pytest config is in my `pyproject.toml` - # so this line is super-simple. But it could be as complex as you need. - - run: | - git config --global user.name "${GITHUB_ACTOR}" - git config --global user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" - poetry run pytest - diff --git a/.lib/git-fleximod/.pre-commit-config.yaml b/.lib/git-fleximod/.pre-commit-config.yaml deleted file mode 100644 index 2f6089da..00000000 --- a/.lib/git-fleximod/.pre-commit-config.yaml +++ /dev/null @@ -1,18 +0,0 @@ -exclude: ^utils/.*$ - -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 - hooks: - - id: end-of-file-fixer - - id: trailing-whitespace - - repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black - - repo: https://github.com/PyCQA/pylint - rev: v2.11.1 - hooks: - - id: pylint - args: - - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error,no-member diff --git a/.lib/git-fleximod/CODE_OF_CONDUCT.md b/.lib/git-fleximod/CODE_OF_CONDUCT.md deleted file mode 100644 index 84f2925b..00000000 --- a/.lib/git-fleximod/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,107 +0,0 @@ -# Contributor Code of Conduct -_The Contributor Code of Conduct is for participants in our software projects and community._ - -## Our Pledge -We, as contributors, creators, stewards, and maintainers (participants), of **git-fleximod** pledge to make participation in -our software, system or hardware project and community a safe, productive, welcoming and inclusive experience for everyone. -All participants are required to abide by this Code of Conduct. -This includes respectful treatment of everyone regardless of age, body size, disability, ethnicity, gender identity or expression, -level of experience, nationality, political affiliation, veteran status, pregnancy, genetic information, physical appearance, race, -religion, or sexual orientation, as well as any other characteristic protected under applicable US federal or state law. - -## Our Standards -Examples of behaviors that contribute to a positive environment include: - -* All participants are treated with respect and consideration, valuing a diversity of views and opinions -* Be considerate, respectful, and collaborative -* Communicate openly with respect for others, critiquing ideas rather than individuals and gracefully accepting criticism -* Acknowledging the contributions of others -* Avoid personal attacks directed toward other participants -* Be mindful of your surroundings and of your fellow participants -* Alert UCAR staff and suppliers/vendors if you notice a dangerous situation or someone in distress -* Respect the rules and policies of the project and venue - -Examples of unacceptable behavior include, but are not limited to: - -* Harassment, intimidation, or discrimination in any form -* Physical, verbal, or written abuse by anyone to anyone, including repeated use of pronouns other than those requested -* Unwelcome sexual attention or advances -* Personal attacks directed at other guests, members, participants, etc. -* Publishing others' private information, such as a physical or electronic address, without explicit permission -* Alarming, intimidating, threatening, or hostile comments or conduct -* Inappropriate use of nudity and/or sexual images -* Threatening or stalking anyone, including a participant -* Other conduct which could reasonably be considered inappropriate in a professional setting - -## Scope -This Code of Conduct applies to all spaces managed by the Project whether they be physical, online or face-to-face. -This includes project code, code repository, associated web pages, documentation, mailing lists, project websites and wiki pages, -issue tracker, meetings, telecons, events, project social media accounts, and any other forums created by the project team which the -community uses for communication. -In addition, violations of this Code of Conduct outside these spaces may affect a person's ability to participate within them. -Representation of a project may be further defined and clarified by project maintainers. - -## Community Responsibilities -Everyone in the community is empowered to respond to people who are showing unacceptable behavior. -They can talk to them privately or publicly. -Anyone requested to stop unacceptable behavior is expected to comply immediately. -If the behavior continues concerns may be brought to the project administrators or to any other party listed in the -[Reporting](#reporting) section below. - -## Project Administrator Responsibilities -Project administrators are responsible for clarifying the standards of acceptable behavior and are encouraged to model appropriate -behavior and provide support when people in the community point out inappropriate behavior. -Project administrator(s) are normally the ones that would be tasked to carry out the actions in the [Consequences](#consequences) -section below. - -Project administrators are also expected to keep this Code of Conduct updated with the main one housed at UCAR, as listed below in -the [Attribution](#attribution) section. - -## Reporting -Instances of unacceptable behavior can be brought to the attention of the project administrator(s) who may take any action as -outlined in the [Consequences](#consequences) section below. -However, making a report to a project administrator is not considered an 'official report' to UCAR. - -Instances of unacceptable behavior may also be reported directly to UCAR pursuant to [UCAR's Harassment Reporting and Complaint -Procedure](https://www2.fin.ucar.edu/procedures/hr/harassment-reporting-and-complaint-procedure), or anonymously through [UCAR's -EthicsPoint Hotline](https://www2.fin.ucar.edu/ethics/anonymous-reporting). - -Complaints received by UCAR will be handled pursuant to the procedures outlined in UCAR's Harassment Reporting and Complaint -Procedure. -Complaints to UCAR will be held as confidential as practicable under the circumstances, and retaliation against a person who -initiates a complaint or an inquiry about inappropriate behavior will not be tolerated. - -Any Contributor can use these reporting methods even if they are not directly affiliated with UCAR. -The Frequently Asked Questions (FAQ) page for reporting is [here](https://www2.fin.ucar.edu/procedures/hr/reporting-faqs). - -## Consequences -Upon receipt of a complaint, the project administrator(s) may take any action deemed necessary and appropriate under the -circumstances. -Such action can include things such as: removing, editing, or rejecting comments, commits, code, wiki edits, email, issues, and -other contributions that are not aligned to this Code of Conduct, or banning temporarily or permanently any contributor for other -behaviors that are deemed inappropriate, threatening, offensive, or harmful. -Project administrators also have the right to report violations to UCAR HR and/or UCAR's Office of Diversity, Equity and Inclusion -(ODEI), as well as a participant's home institution and/or law enforcement. -In the event an incident is reported to UCAR, UCAR will follow its Harassment Reporting and Complaint Procedure. - -## Process for Changes -All UCAR managed projects are required to adopt this Contributor Code of Conduct. -Adoption is assumed even if not expressly stated in the repository. -Projects should fill in sections where prompted with project-specific information, including, project name and adoption date. - -Projects that adopt this Code of Conduct need to stay up to date with UCAR's Contributor Code of Conduct, linked with a DOI in the -[Attribution](#attribution) section below. -Projects can make limited substantive changes to the Code of Conduct, however, the changes must be limited in scope and may not -contradict the UCAR Contributor Code of Conduct. - -## Attribution -This Code of Conduct was originally adapted from the [Contributor Covenant](http://contributor-covenant.org/version/1/4), version -1.4. -We then aligned it with the UCAR Participant Code of Conduct, which also borrows from the American Geophysical Union (AGU) Code of -Conduct. -The UCAR Participant Code of Conduct applies to both UCAR employees as well as participants in activities run by UCAR. -The original version of this for all software projects that have strong management from UCAR or UCAR staff is available on the UCAR -website at https://doi.org/10.5065/6w2c-a132. -The date that it was adopted by this project was **Feb/13/2018**. -When responding to complaints, UCAR HR and ODEI will do so based on the latest published version. -Therefore, any project-specific changes should follow the [Process for Changes](#process-for-changes) section above. diff --git a/.lib/git-fleximod/License b/.lib/git-fleximod/License deleted file mode 100644 index 88bc2251..00000000 --- a/.lib/git-fleximod/License +++ /dev/null @@ -1,20 +0,0 @@ -Copyright 2024 NSF National Center for Atmospheric Sciences (NCAR) - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -“Software”), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/.lib/git-fleximod/README.md b/.lib/git-fleximod/README.md deleted file mode 100644 index 53917da4..00000000 --- a/.lib/git-fleximod/README.md +++ /dev/null @@ -1,108 +0,0 @@ -# git-fleximod - -Flexible, Enhanced Submodule Management for Git - -## Overview - -Git-fleximod is a Python-based tool that extends Git's submodule and sparse checkout capabilities, offering additional features for managing submodules in a more flexible and efficient way. - -## Installation - - If you choose to locate git-fleximod in your path you can access it via command: git fleximod - -## Usage - - Basic Usage: - git fleximod [options] - Available Commands: - status: Display the status of submodules. - update: Update submodules to the tag indicated in .gitmodules variable fxtag. - test: Make sure that fxtags and submodule hashes are consistant, - make sure that official urls (as defined by fxDONOTUSEurl) are set - make sure that fxtags are defined for all submodules - Additional Options: - See git fleximod --help for more details. - -## Supported .gitmodules Variables - - fxtag: Specify a specific tag or branch to checkout for a submodule. - fxrequired: Mark a submodule's checkout behavior, with allowed values: - - ToplevelRequired: Top-level and required (checked out only when this is the Toplevel module). - - ToplevelOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module). - - AlwaysRequired: Always required (always checked out). - - AlwaysOptional: Always optional (checked out with --optional flag). - fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths. - fxDONOTUSEurl: This is the url used in the test subcommand to assure that protected branches do not point to forks - **NOTE** the fxDONOTUSEurl variable is only used to identify the official project repository and should not be - changed by users. Use the url variable to change to a fork if desired. - -## Sparse Checkouts - - To enable sparse checkout for a submodule, set the fxsparse variable - in the .gitmodules file to the path of a file containing the desired - sparse checkout paths. Git-fleximod will automatically configure - sparse checkout based on this file when applicable commands are run. - See [git-sparse-checkout](https://git-scm.com/docs/git-sparse-checkout#_internalsfull_pattern_set) - for details on the format of this file. - -## Tests - - The git fleximod test action is designed to be used by, for example, github workflows - to assure that protected branches are consistant with respect to submodule hashes and fleximod fxtags - -## Examples - -Here are some common usage examples: - -Update all submodules, including optional ones: -```bash - git fleximod update --optional -``` - -Updating a specific submodule to the fxtag indicated in .gitmodules: - -```bash - git fleximod update submodule-name -``` -Example .gitmodules entry: -```ini, toml - [submodule "cosp2"] - path = src/physics/cosp2/src - url = https://github.com/CFMIP/COSPv2.0 - fxsparse = ../.cosp_sparse_checkout - fxrequired = AlwaysRequired - fxtag = v2.1.4cesm -``` -Explanation: - -This entry indicates that the submodule named cosp2 at tag v2.1.4cesm -should be checked out into the directory src/physics/cosp2/src -relative to the .gitmodules directory. It should be checked out from -the URL https://github.com/CFMIP/COSPv2.0 and use sparse checkout as -described in the file ../.cosp_sparse_checkout relative to the path -directory. It should be checked out anytime this .gitmodules entry is -read. - -Additional example: -```ini, toml - [submodule "cime"] - path = cime - url = https://github.com/jedwards4b/cime - fxrequired = ToplevelRequired - fxtag = cime6.0.198_rme01 -``` - -Explanation: - -This entry indicates that the submodule cime should be checked out -into a directory named cime at tag cime6.0.198_rme01 from the URL -https://github.com/jedwards4b/cime. This should only be done if -the .gitmodules file is at the top level of the repository clone. - -## Contributing - -We welcome contributions! Please see the CONTRIBUTING.md file for guidelines. - -## License - -Git-fleximod is released under the MIT License. diff --git a/.lib/git-fleximod/doc/Makefile b/.lib/git-fleximod/doc/Makefile deleted file mode 100644 index d4bb2cbb..00000000 --- a/.lib/git-fleximod/doc/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/.lib/git-fleximod/doc/conf.py b/.lib/git-fleximod/doc/conf.py deleted file mode 100644 index 423099ee..00000000 --- a/.lib/git-fleximod/doc/conf.py +++ /dev/null @@ -1,26 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# For the full list of built-in configuration values, see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Project information ----------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information - -project = "git-fleximod" -author = "Jim Edwards " -release = "0.4.0" - -# -- General configuration --------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration - -extensions = ["sphinx_argparse_cli"] - -templates_path = ["_templates"] -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - - -# -- Options for HTML output ------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output - -html_theme = "alabaster" -html_static_path = ["_static"] diff --git a/.lib/git-fleximod/doc/index.rst b/.lib/git-fleximod/doc/index.rst deleted file mode 100644 index 0f9c1a7f..00000000 --- a/.lib/git-fleximod/doc/index.rst +++ /dev/null @@ -1,24 +0,0 @@ -.. git-fleximod documentation master file, created by - sphinx-quickstart on Sat Feb 3 12:02:22 2024. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to git-fleximod's documentation! -======================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: -.. module:: sphinxcontrib.autoprogram -.. sphinx_argparse_cli:: - :module: git_fleximod.cli - :func: get_parser - :prog: git-fleximod - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/.lib/git-fleximod/doc/make.bat b/.lib/git-fleximod/doc/make.bat deleted file mode 100644 index 32bb2452..00000000 --- a/.lib/git-fleximod/doc/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "" goto help - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/.lib/git-fleximod/escomp_install b/.lib/git-fleximod/escomp_install deleted file mode 100644 index ae782e72..00000000 --- a/.lib/git-fleximod/escomp_install +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# updates git-fleximod in an ESCOMP model -# this script should be run from the model root directory, it expects -# git-fleximod to already be installed with the script in bin -# and the classes in lib/python/site-packages -import sys -import shutil -import os - -from glob import iglob - -fleximod_root = sys.argv[1] -fleximod_path = os.path.join(fleximod_root,"src","git-fleximod") -if os.path.isfile(fleximod_path): - with open(fleximod_path,"r") as f: - fleximod = f.readlines() - with open(os.path.join(".","bin","git-fleximod"),"w") as f: - for line in fleximod: - f.write(line) - if "import argparse" in line: - f.write('\nsys.path.append(os.path.join(os.path.dirname(__file__),"..","lib","python","site-packages"))\n\n') - - for file in iglob(os.path.join(fleximod_root, "src", "fleximod", "*.py")): - shutil.copy(file, - os.path.join("lib","python","site-packages","fleximod",os.path.basename(file))) diff --git a/.lib/git-fleximod/git_fleximod/__init__.py b/.lib/git-fleximod/git_fleximod/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/.lib/git-fleximod/git_fleximod/cli.py b/.lib/git-fleximod/git_fleximod/cli.py deleted file mode 100644 index b5a05494..00000000 --- a/.lib/git-fleximod/git_fleximod/cli.py +++ /dev/null @@ -1,145 +0,0 @@ -from pathlib import Path -import argparse, os, sys -from git_fleximod import utils - -__version__ = "1.0.2" - -class CustomArgumentParser(argparse.ArgumentParser): - def print_help(self, file=None): - # First print the default help message - super().print_help(file) - - # Then append the contents of README.md - candidate_paths = [ - Path(sys.prefix) / "share" / "git_fleximod" / "README.md", - Path(__file__).resolve().parent.parent / "README.md", # fallback for dev - ] - for path in candidate_paths: - if os.path.exists(path): - with open(path) as f: - print( f.read(), file=file) - return - print( "README.md not found.", file=file) - -def find_root_dir(filename=".gitmodules"): - """ finds the highest directory in tree - which contains a file called filename """ - d = Path.cwd() - root = Path(d.root) - dirlist = [] - dl = d - while dl != root: - dirlist.append(dl) - dl = dl.parent - dirlist.append(root) - dirlist.reverse() - - for dl in dirlist: - attempt = dl / filename - if attempt.is_file(): - return str(dl) - return None - -def get_parser(): - description = """ - %(prog)s manages checking out groups of gitsubmodules with additional support for Earth System Models - """ - parser = CustomArgumentParser( - description=description, formatter_class=argparse.RawDescriptionHelpFormatter - ) - - # - # user options - # - choices = ["update", "status", "test"] - parser.add_argument( - "action", - choices=choices, - default="update", - help=f"Subcommand of git-fleximod, choices are {choices[:-1]}", - ) - - parser.add_argument( - "components", - nargs="*", - help="Specific component(s) to checkout. By default, " - "all required submodules are checked out.", - ) - - parser.add_argument( - "-C", - "--path", - default=find_root_dir(), - help="Toplevel repository directory. Defaults to top git directory relative to current.", - ) - - parser.add_argument( - "-g", - "--gitmodules", - nargs="?", - default=".gitmodules", - help="The submodule description filename. " "Default: %(default)s.", - ) - - parser.add_argument( - "-x", - "--exclude", - nargs="*", - help="Component(s) listed in the gitmodules file which should be ignored.", - ) - parser.add_argument( - "-f", - "--force", - action="store_true", - default=False, - help="Override cautions and update or checkout over locally modified repository.", - ) - - parser.add_argument( - "-o", - "--optional", - action="store_true", - default=False, - help="By default only the required submodules " - "are checked out. This flag will also checkout the " - "optional submodules relative to the toplevel directory.", - ) - - parser.add_argument( - "-v", - "--verbose", - action="count", - default=0, - help="Output additional information to " - "the screen and log file. This flag can be " - "used up to two times, increasing the " - "verbosity level each time.", - ) - - parser.add_argument( - "-V", - "--version", - action="version", - version=f"%(prog)s {__version__}", - help="Print version and exit.", - ) - - # - # developer options - # - parser.add_argument( - "--backtrace", - action="store_true", - help="DEVELOPER: show exception backtraces as extra " "debugging output", - ) - - parser.add_argument( - "-d", - "--debug", - action="store_true", - default=False, - help="DEVELOPER: output additional debugging " - "information to the screen and log file.", - ) - - return parser diff --git a/.lib/git-fleximod/git_fleximod/git_fleximod.py b/.lib/git-fleximod/git_fleximod/git_fleximod.py deleted file mode 100755 index b3c4fece..00000000 --- a/.lib/git-fleximod/git_fleximod/git_fleximod.py +++ /dev/null @@ -1,370 +0,0 @@ -#!/usr/bin/env python -import sys - -MIN_PYTHON = (3, 7) -if sys.version_info < MIN_PYTHON: - sys.exit("Python %s.%s or later is required." % MIN_PYTHON) - -import os -import shutil -import logging -import textwrap -import asyncio -from git_fleximod import utils -from git_fleximod import cli -from git_fleximod.gitinterface import GitInterface -from git_fleximod.gitmodules import GitModules -from git_fleximod.submodule import Submodule - -# logger variable is global -logger = None - - -def fxrequired_allowed_values(): - return ["ToplevelRequired", "ToplevelOptional", "AlwaysRequired", "AlwaysOptional", "TopLevelRequired", "TopLevelOptional"] - - -def commandline_arguments(args=None): - parser = cli.get_parser() - - if args: - options = parser.parse_args(args) - else: - options = parser.parse_args() - - # explicitly listing a component overrides the optional flag - if options.optional or options.components: - fxrequired = fxrequired_allowed_values() - else: - fxrequired = ["ToplevelRequired", "AlwaysRequired", "TopLevelRequired"] - - action = options.action - if not action: - action = "update" - handlers = [logging.StreamHandler()] - - if options.debug: - try: - open("fleximod.log", "w") - except PermissionError: - sys.exit("ABORT: Could not write file fleximod.log") - level = logging.DEBUG - handlers.append(logging.FileHandler("fleximod.log")) - elif options.verbose: - level = logging.INFO - else: - level = logging.WARNING - # Configure the root logger - logging.basicConfig( - level=level, format="%(name)s - %(levelname)s - %(message)s", handlers=handlers - ) - - if hasattr(options, "version"): - exit() - - return ( - options.path, - options.gitmodules, - fxrequired, - options.components, - options.exclude, - options.force, - action, - ) - - -def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master"): - """ - This function performs a sparse checkout of a git submodule. It does so by first creating the .git/info/sparse-checkout fileq - in the submodule and then checking out the desired tag. If the submodule is already checked out, it will not be checked out again. - Creating the sparse-checkout file first prevents the entire submodule from being checked out and then removed. This is important - because the submodule may have a large number of files and checking out the entire submodule and then removing it would be time - and disk space consuming. - - Parameters: - root_dir (str): The root directory for the git operation. - name (str): The name of the submodule. - url (str): The URL of the submodule. - path (str): The path to the submodule. - sparsefile (str): The sparse file for the submodule. - tag (str, optional): The tag to checkout. Defaults to "master". - - Returns: - None - """ - logger.info("Called sparse_checkout for {}".format(name)) - rgit = GitInterface(root_dir, logger) - superroot = git_toplevelroot(root_dir, logger) - - if superroot: - gitroot = superroot.strip() - else: - gitroot = root_dir.strip() - assert os.path.isdir(os.path.join(gitroot, ".git")) - # first create the module directory - if not os.path.isdir(os.path.join(root_dir, path)): - os.makedirs(os.path.join(root_dir, path)) - - # initialize a new git repo and set the sparse checkout flag - sprep_repo = os.path.join(root_dir, path) - sprepo_git = GitInterface(sprep_repo, logger) - if os.path.exists(os.path.join(sprep_repo, ".git")): - try: - logger.info("Submodule {} found".format(name)) - chk = sprepo_git.config_get_value("core", "sparseCheckout") - if chk == "true": - logger.info("Sparse submodule {} already checked out".format(name)) - return - except NoOptionError: - logger.debug("Sparse submodule {} not present".format(name)) - except Exception as e: - utils.fatal_error("Unexpected error {} occured.".format(e)) - - sprepo_git.config_set_value("core", "sparseCheckout", "true") - - # set the repository remote - - logger.info("Setting remote origin in {}/{}".format(root_dir, path)) - _, remotelist = sprepo_git.git_operation("remote", "-v") - if url not in remotelist: - sprepo_git.git_operation("remote", "add", "origin", url) - - topgit = os.path.join(gitroot, ".git") - - if gitroot != root_dir and os.path.isfile(os.path.join(root_dir, ".git")): - with open(os.path.join(root_dir, ".git")) as f: - gitpath = os.path.relpath( - os.path.join(root_dir, f.read().split()[1]), - start=os.path.join(root_dir, path), - ) - topgit = os.path.join(gitpath, "modules") - else: - topgit = os.path.relpath( - os.path.join(root_dir, ".git", "modules"), - start=os.path.join(root_dir, path), - ) - - with utils.pushd(sprep_repo): - if not os.path.isdir(topgit): - os.makedirs(topgit) - topgit += os.sep + name - - if os.path.isdir(os.path.join(root_dir, path, ".git")): - with utils.pushd(sprep_repo): - if os.path.isdir(os.path.join(topgit,".git")): - shutil.rmtree(os.path.join(topgit,".git")) - shutil.move(".git", topgit) - with open(".git", "w") as f: - f.write("gitdir: " + os.path.relpath(topgit)) - # assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo))) - gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout")) - if os.path.isfile(gitsparse): - logger.warning( - "submodule {} is already initialized {}".format(name, topgit) - ) - return - - with utils.pushd(sprep_repo): - if os.path.isfile(sparsefile): - shutil.copy(sparsefile, gitsparse) - - - # Finally checkout the repo - sprepo_git.git_operation("fetch", "origin", "--tags") - sprepo_git.git_operation("checkout", tag) - - print(f"Successfully checked out {name:>20} at {tag}") - rgit.config_set_value(f'submodule "{name}"', "active", "true") - rgit.config_set_value(f'submodule "{name}"', "url", url) - -def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger): - path = gitmodules.get(name, "path") - url = gitmodules.get(name, "url") - assert path and url, f"Malformed .gitmodules file {path} {url}" - tag = gitmodules.get(name, "fxtag") - if not tag: - tag = gitmodules.get(name, "hash") - fxurl = gitmodules.get(name, "fxDONOTUSEurl") - fxsparse = gitmodules.get(name, "fxsparse") - fxrequired = gitmodules.get(name, "fxrequired") - return Submodule(root_dir, name, path, url, fxtag=tag, fxurl=fxurl, fxsparse=fxsparse, fxrequired=fxrequired, logger=logger) - -def submodules_status(gitmodules, root_dir, toplevel=False, depth=0): - testfails = 0 - localmods = 0 - needsupdate = 0 - wrapper = textwrap.TextWrapper(initial_indent=' '*(depth*10), width=120,subsequent_indent=' '*(depth*20)) - for name in gitmodules.sections(): - submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) - - result,n,l,t = submod.status() - if toplevel or not submod.toplevel(): - print(wrapper.fill(result)) - testfails += t - localmods += l - needsupdate += n - subdir = os.path.join(root_dir, submod.path) - if os.path.exists(os.path.join(subdir, ".gitmodules")): - gsubmod = GitModules(logger, confpath=subdir) - t,l,n = submodules_status(gsubmod, subdir, depth=depth+1) - if toplevel or not submod.toplevel(): - testfails += t - localmods += l - needsupdate += n - - return testfails, localmods, needsupdate - -def git_toplevelroot(root_dir, logger): - rgit = GitInterface(root_dir, logger) - _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") - return superroot - -async def submodules_update(gitmodules, root_dir, requiredlist, force): - async def update_submodule(name, requiredlist, force): - submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) - - _, needsupdate, localmods, testfails = submod.status() - if not submod.fxrequired: - submod.fxrequired = "AlwaysRequired" - fxrequired = submod.fxrequired - allowedvalues = fxrequired_allowed_values() - assert fxrequired in allowedvalues - - superroot = git_toplevelroot(root_dir, logger) - - if ( - fxrequired - and ((superroot and "Toplevel" in fxrequired) - or fxrequired not in requiredlist) - ): - if "Optional" in fxrequired and "Optional" not in requiredlist: - if fxrequired.startswith("Always"): - print(f"Skipping optional component {name:>20}") - return # continue to next submodule - optional = "AlwaysOptional" in requiredlist - - if fxrequired in requiredlist: - await submod.update() - repodir = os.path.join(root_dir, submod.path) - if os.path.exists(os.path.join(repodir, ".gitmodules")): - # recursively handle this checkout - print(f"Recursively checking out submodules of {name}") - gitsubmodules = GitModules(submod.logger, confpath=repodir) - newrequiredlist = ["AlwaysRequired"] - if optional: - newrequiredlist.append("AlwaysOptional") - await submodules_update(gitsubmodules, repodir, newrequiredlist, force=force) - - tasks = [update_submodule(name, requiredlist, force) for name in gitmodules.sections()] - await asyncio.gather(*tasks) - -def local_mods_output(): - text = """\ - The submodules labeled with 'M' above are not in a clean state. - The following are options for how to proceed: - (1) Go into each submodule which is not in a clean state and issue a 'git status' - Either revert or commit your changes so that the submodule is in a clean state. - (2) use the --force option to git-fleximod - (3) you can name the particular submodules to update using the git-fleximod command line - (4) As a last resort you can remove the submodule (via 'rm -fr [directory]') - then rerun git-fleximod update. -""" - print(text) - -def submodules_test(gitmodules, root_dir): - """ - This function tests the git submodules based on the provided parameters. - - It first checks that fxtags are present and in sync with submodule hashes. - Then it ensures that urls are consistent with fxurls (not forks and not ssh) - and that sparse checkout files exist. - - Parameters: - gitmodules (ConfigParser): The gitmodules configuration. - root_dir (str): The root directory for the git operation. - - Returns: - int: The number of test failures. - """ - # First check that fxtags are present and in sync with submodule hashes - testfails, localmods, needsupdate = submodules_status(gitmodules, root_dir) - print("") - # Then make sure that urls are consistant with fxurls (not forks and not ssh) - # and that sparse checkout files exist - for name in gitmodules.sections(): - url = gitmodules.get(name, "url") - fxurl = gitmodules.get(name, "fxDONOTUSEurl") - fxsparse = gitmodules.get(name, "fxsparse") - path = gitmodules.get(name, "path") - fxurl = fxurl[:-4] if fxurl.endswith(".git") else fxurl - url = url[:-4] if url.endswith(".git") else url - if not fxurl or url.lower() != fxurl.lower(): - print(f"{name:>20} url {url} not in sync with required {fxurl}") - testfails += 1 - if fxsparse and not os.path.isfile(os.path.join(root_dir, path, fxsparse)): - print(f"{name:>20} sparse checkout file {fxsparse} not found") - testfails += 1 - return testfails + localmods + needsupdate - - -def main(): - ( - root_dir, - file_name, - fxrequired, - includelist, - excludelist, - force, - action, - ) = commandline_arguments() - # Get a logger for the package - global logger - logger = logging.getLogger(__name__) - - logger.info("action is {} root_dir={} file_name={}".format(action, root_dir, file_name)) - - if not root_dir or not os.path.isfile(os.path.join(root_dir, file_name)): - if root_dir: - file_path = utils.find_upwards(root_dir, file_name) - - if root_dir is None or file_path is None: - root_dir = "." - utils.fatal_error( - "No {} found in {} or any of it's parents".format(file_name, root_dir) - ) - - root_dir = os.path.dirname(file_path) - logger.info( - "root_dir is {} includelist={} excludelist={}".format( - root_dir, includelist, excludelist - ) - ) - gitmodules = GitModules( - logger, - confpath=root_dir, - conffile=file_name, - includelist=includelist, - excludelist=excludelist, - ) - if not gitmodules.sections(): - sys.exit(f"No submodule components found, root_dir={root_dir}") - retval = 0 - if action == "update": - asyncio.run(submodules_update(gitmodules, root_dir, fxrequired, force)) - elif action == "status": - tfails, lmods, updates = submodules_status(gitmodules, root_dir, toplevel=True) - if tfails + lmods + updates > 0: - print( - f" testfails = {tfails}, local mods = {lmods}, needs updates {updates}\n" - ) - if lmods > 0: - local_mods_output() - elif action == "test": - retval = submodules_test(gitmodules, root_dir) - else: - utils.fatal_error(f"unrecognized action request {action}") - return retval - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.lib/git-fleximod/git_fleximod/gitinterface.py b/.lib/git-fleximod/git_fleximod/gitinterface.py deleted file mode 100644 index a24870a5..00000000 --- a/.lib/git-fleximod/git_fleximod/gitinterface.py +++ /dev/null @@ -1,126 +0,0 @@ -import os -import sys -from . import utils -from pathlib import Path -import asyncio - -class GitInterface: - def __init__(self, repo_path, logger): - logger.debug("Initialize GitInterface for {}".format(repo_path)) - if isinstance(repo_path, str): - self.repo_path = Path(repo_path).resolve() - elif isinstance(repo_path, Path): - self.repo_path = repo_path.resolve() - else: - raise TypeError("repo_path must be a str or Path object") - self.logger = logger - try: - import git - - self._use_module = True - try: - self.repo = git.Repo(str(self.repo_path)) # Initialize GitPython repo - except git.exc.InvalidGitRepositoryError: - self.git = git - self._init_git_repo() - msg = "Using GitPython interface to git" - except ImportError: - self._use_module = False - if not (self.repo_path / ".git").exists(): - self._init_git_repo() - msg = "Using shell interface to git" - self.logger.info(msg) - - def _git_command(self, operation, *args): - self.logger.info(operation) - if self._use_module and operation != "submodule": - try: - return getattr(self.repo.git, operation)(*args) - except Exception as e: - sys.exit(e) - else: - return ["git", "-C", str(self.repo_path), operation] + list(args) - - def _init_git_repo(self): - if self._use_module: - self.repo = self.git.Repo.init(str(self.repo_path)) - else: - command = ("git", "-C", str(self.repo_path), "init") - utils.execute_subprocess(command) - - def _git_operation_command(self, operation, args): - newargs = [] - for a in args: - # Do not use ssh interface - if isinstance(a, str): - a = a.replace("git@github.com:", "https://github.com/") - newargs.append(a) - - return self._git_command(operation, *newargs) - - # pylint: disable=unused-argument - def git_operation(self, operation, *args, **kwargs): - newargs = [] - for a in args: - # Do not use ssh interface - if isinstance(a, str): - a = a.replace("git@github.com:", "https://github.com/") - newargs.append(a) - - return self._git_command(operation, *newargs) - - # pylint: disable=unused-argument - def git_operation(self, operation, *args, **kwargs): - command = self._git_operation_command(operation, args) - if isinstance(command, list): - try: - status, output = utils.execute_subprocess(command, status_to_caller=True, output_to_caller=True) - return status, output.rstrip() - except Exception as e: - sys.exit(e) - else: - return 0, command - - # pylint: disable=unused-argument - async def git_operation_async(self, operation, *args, **kwargs): - command = self._git_operation_command(operation, args) - if isinstance(command, list): - try: - process = await asyncio.create_subprocess_exec( - *command, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE - ) - stdout, stderr = await process.communicate() - status = process.returncode - output = stdout.decode().strip() if stdout else stderr.decode().strip() - return status, output - except Exception as e: - sys.exit(e) - else: - return 0, command - - def config_get_value(self, section, name): - if self._use_module: - config = self.repo.config_reader() - try: - val = config.get_value(section, name) - except: - val = None - return val - else: - cmd = ("git", "-C", str(self.repo_path), "config", "--get", f"{section}.{name}") - output = utils.execute_subprocess(cmd, output_to_caller=True) - return output.strip() - - def config_set_value(self, section, name, value): - if self._use_module: - with self.repo.config_writer() as writer: - if "." in section: - section = section.replace("."," \"")+'"' - writer.set_value(section, name, value) - writer.release() # Ensure changes are saved - else: - cmd = ("git", "-C", str(self.repo_path), "config", f"{section}.{name}", value) - self.logger.info(cmd) - utils.execute_subprocess(cmd, output_to_caller=True) diff --git a/.lib/git-fleximod/git_fleximod/gitmodules.py b/.lib/git-fleximod/git_fleximod/gitmodules.py deleted file mode 100644 index cf8b350d..00000000 --- a/.lib/git-fleximod/git_fleximod/gitmodules.py +++ /dev/null @@ -1,97 +0,0 @@ -import shutil, os -from pathlib import Path -from configparser import RawConfigParser, ConfigParser -from .lstripreader import LstripReader - - -class GitModules(RawConfigParser): - def __init__( - self, - logger, - confpath=Path.cwd(), - conffile=".gitmodules", - includelist=None, - excludelist=None, - ): - """ - confpath: Path to the directory containing the .gitmodules file (defaults to the current working directory). - conffile: Name of the configuration file (defaults to .gitmodules). - includelist: Optional list of submodules to include. - excludelist: Optional list of submodules to exclude. - """ - self.logger = logger - self.logger.debug( - "Creating a GitModules object {} {} {} {}".format( - confpath, conffile, includelist, excludelist - ) - ) - super().__init__() - self.conf_file = (Path(confpath) / Path(conffile)) - if self.conf_file.exists(): - self.read_file(LstripReader(str(self.conf_file)), source=conffile) - self.includelist = includelist - self.excludelist = excludelist - self.isdirty = False - - def reload(self): - self.clear() - if self.conf_file.exists(): - self.read_file(LstripReader(str(self.conf_file)), source=self.conf_file) - - - def set(self, name, option, value): - """ - Sets a configuration value for a specific submodule: - Ensures the appropriate section exists for the submodule. - Calls the parent class's set method to store the value. - """ - self.isdirty = True - self.logger.debug("set called {} {} {}".format(name, option, value)) - section = f'submodule "{name}"' - if not self.has_section(section): - self.add_section(section) - super().set(section, option, str(value)) - - # pylint: disable=redefined-builtin, arguments-differ - def get(self, name, option, raw=False, vars=None, fallback=None): - """ - Retrieves a configuration value for a specific submodule: - Uses the parent class's get method to access the value. - Handles potential errors if the section or option doesn't exist. - """ - self.logger.debug("git get called {} {}".format(name, option)) - section = f'submodule "{name}"' - try: - return ConfigParser.get( - self, section, option, raw=raw, vars=vars, fallback=fallback - ) - except ConfigParser.NoOptionError: - return None - - def save(self): - if self.isdirty: - self.logger.info("Writing {}".format(self.conf_file)) - with open(self.conf_file, "w") as fd: - self.write(fd) - self.isdirty = False - - def __del__(self): - self.save() - - def sections(self): - """Strip the submodule part out of section and just use the name""" - self.logger.debug("calling GitModules sections iterator") - names = [] - for section in ConfigParser.sections(self): - name = section[11:-1] - if self.includelist and name not in self.includelist: - continue - if self.excludelist and name in self.excludelist: - continue - names.append(name) - return names - - def items(self, name, raw=False, vars=None): - self.logger.debug("calling GitModules items for {}".format(name)) - section = f'submodule "{name}"' - return ConfigParser.items(section, raw=raw, vars=vars) diff --git a/.lib/git-fleximod/git_fleximod/lstripreader.py b/.lib/git-fleximod/git_fleximod/lstripreader.py deleted file mode 100644 index 01d5580e..00000000 --- a/.lib/git-fleximod/git_fleximod/lstripreader.py +++ /dev/null @@ -1,43 +0,0 @@ -class LstripReader(object): - "LstripReader formats .gitmodules files to be acceptable for configparser" - - def __init__(self, filename): - with open(filename, "r") as infile: - lines = infile.readlines() - self._lines = list() - self._num_lines = len(lines) - self._index = 0 - for line in lines: - self._lines.append(line.lstrip()) - - def readlines(self): - """Return all the lines from this object's file""" - return self._lines - - def readline(self, size=-1): - """Format and return the next line or raise StopIteration""" - try: - line = self.next() - except StopIteration: - line = "" - - if (size > 0) and (len(line) < size): - return line[0:size] - - return line - - def __iter__(self): - """Begin an iteration""" - self._index = 0 - return self - - def next(self): - """Return the next line or raise StopIteration""" - if self._index >= self._num_lines: - raise StopIteration - - self._index = self._index + 1 - return self._lines[self._index - 1] - - def __next__(self): - return self.next() diff --git a/.lib/git-fleximod/git_fleximod/metoflexi.py b/.lib/git-fleximod/git_fleximod/metoflexi.py deleted file mode 100755 index cc347db2..00000000 --- a/.lib/git-fleximod/git_fleximod/metoflexi.py +++ /dev/null @@ -1,236 +0,0 @@ -#!/usr/bin/env python -from configparser import ConfigParser -import sys -import shutil -from pathlib import Path -import argparse -import logging -from git_fleximod.gitinterface import GitInterface -from git_fleximod.gitmodules import GitModules -from git_fleximod import utils - -logger = None - -def find_root_dir(filename=".git"): - d = Path.cwd() - root = Path(d.root) - while d != root: - attempt = d / filename - if attempt.is_dir(): - return d - d = d.parent - return None - - -def get_parser(): - description = """ - %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models - """ - parser = argparse.ArgumentParser( - description=description, formatter_class=argparse.RawDescriptionHelpFormatter - ) - - parser.add_argument('-e', '--externals', nargs='?', - default='Externals.cfg', - help='The externals description filename. ' - 'Default: %(default)s.') - - parser.add_argument( - "-C", - "--path", - default=find_root_dir(), - help="Toplevel repository directory. Defaults to top git directory relative to current.", - ) - - parser.add_argument( - "-g", - "--gitmodules", - nargs="?", - default=".gitmodules", - help="The submodule description filename. " "Default: %(default)s.", - ) - parser.add_argument( - "-v", - "--verbose", - action="count", - default=0, - help="Output additional information to " - "the screen and log file. This flag can be " - "used up to two times, increasing the " - "verbosity level each time.", - ) - parser.add_argument( - "-d", - "--debug", - action="store_true", - default=False, - help="DEVELOPER: output additional debugging " - "information to the screen and log file.", - ) - - return parser - -def commandline_arguments(args=None): - parser = get_parser() - - options = parser.parse_args(args) - handlers = [logging.StreamHandler()] - - if options.debug: - try: - open("fleximod.log", "w") - except PermissionError: - sys.exit("ABORT: Could not write file fleximod.log") - level = logging.DEBUG - handlers.append(logging.FileHandler("fleximod.log")) - elif options.verbose: - level = logging.INFO - else: - level = logging.WARNING - # Configure the root logger - logging.basicConfig( - level=level, format="%(name)s - %(levelname)s - %(message)s", handlers=handlers - ) - - return( - options.path, - options.gitmodules, - options.externals - ) - -class ExternalRepoTranslator: - """ - Translates external repositories configured in an INI-style externals file. - """ - - def __init__(self, rootpath, gitmodules, externals): - self.rootpath = rootpath - if gitmodules: - self.gitmodules = GitModules(logger, confpath=rootpath) - self.externals = (rootpath / Path(externals)).resolve() - print(f"Translating {self.externals}") - self.git = GitInterface(rootpath, logger) - -# def __del__(self): -# if (self.rootpath / "save.gitignore"): - - - def translate_single_repo(self, section, tag, url, path, efile, hash_, sparse, protocol): - """ - Translates a single repository based on configuration details. - - Args: - rootpath (str): Root path of the main repository. - gitmodules (str): Path to the .gitmodules file. - tag (str): The tag to use for the external repository. - url (str): The URL of the external repository. - path (str): The relative path within the main repository for the external repository. - efile (str): The external file or file containing submodules. - hash_ (str): The commit hash to checkout (if applicable). - sparse (str): Boolean indicating whether to use sparse checkout (if applicable). - protocol (str): The protocol to use (e.g., 'git', 'http'). - """ - assert protocol != "svn", "SVN protocol is not currently supported" - print(f"Translating repository {section}") - if efile: - file_path = Path(path) / Path(efile) - newroot = (self.rootpath / file_path).parent.resolve() - if not newroot.exists(): - newroot.mkdir(parents=True) - logger.info("Newroot is {}".format(newroot)) - newt = ExternalRepoTranslator(newroot, ".gitmodules", efile) - newt.translate_repo() - if protocol == "externals_only": - if tag: - self.gitmodules.set(section, "fxtag", tag) - if hash_: - self.gitmodules.set(section, "fxtag", hash_) - - self.gitmodules.set(section, "fxDONOTUSEurl", url) - if sparse: - self.gitmodules.set(section, "fxsparse", sparse) - self.gitmodules.set(section, "fxrequired", "ToplevelRequired") - else: - newpath = (self.rootpath / Path(path)) - if newpath.exists(): - shutil.rmtree(newpath) - logger.info("Creating directory {}".format(newpath)) - newpath.mkdir(parents=True) - if tag: - logger.info("cloning {}".format(section)) - try: - self.git.git_operation("clone", "-b", tag, "--depth", "1", url, path) - except: - self.git.git_operation("clone", url, path) - with utils.pushd(newpath): - ngit = GitInterface(newpath, logger) - ngit.git_operation("checkout", tag) - if hash_: - self.git.git_operation("clone", url, path) - git = GitInterface(newpath, logger) - git.git_operation("fetch", "origin") - git.git_operation("checkout", hash_) - if sparse: - print("setting as sparse submodule {}".format(section)) - sparsefile = (newpath / Path(sparse)) - newfile = (newpath / ".git" / "info" / "sparse-checkout") - print(f"sparsefile {sparsefile} newfile {newfile}") - shutil.copy(sparsefile, newfile) - - logger.info("adding submodule {}".format(section)) - self.gitmodules.save() - self.git.git_operation("submodule", "add", "-f", "--name", section, url, path) - self.git.git_operation("submodule","absorbgitdirs") - self.gitmodules.reload() - if tag: - self.gitmodules.set(section, "fxtag", tag) - if hash_: - self.gitmodules.set(section, "fxtag", hash_) - - self.gitmodules.set(section, "fxDONOTUSEurl", url) - if sparse: - self.gitmodules.set(section, "fxsparse", sparse) - self.gitmodules.set(section, "fxrequired", "ToplevelRequired") - - - def translate_repo(self): - """ - Translates external repositories defined within an external file. - - Args: - rootpath (str): Root path of the main repository. - gitmodules (str): Path to the .gitmodules file. - external_file (str): The path to the external file containing repository definitions. - """ - econfig = ConfigParser() - econfig.read((self.rootpath / Path(self.externals))) - - for section in econfig.sections(): - if section == "externals_description": - logger.info("skipping section {}".format(section)) - return - logger.info("Translating section {}".format(section)) - tag = econfig.get(section, "tag", raw=False, fallback=None) - url = econfig.get(section, "repo_url", raw=False, fallback=None) - path = econfig.get(section, "local_path", raw=False, fallback=None) - efile = econfig.get(section, "externals", raw=False, fallback=None) - hash_ = econfig.get(section, "hash", raw=False, fallback=None) - sparse = econfig.get(section, "sparse", raw=False, fallback=None) - protocol = econfig.get(section, "protocol", raw=False, fallback=None) - - self.translate_single_repo(section, tag, url, path, efile, hash_, sparse, protocol) - - - -def _main(): - rootpath, gitmodules, externals = commandline_arguments() - global logger - logger = logging.getLogger(__name__) - with utils.pushd(rootpath): - t = ExternalRepoTranslator(Path(rootpath), gitmodules, externals) - logger.info("Translating {}".format(rootpath)) - t.translate_repo() - - -if __name__ == "__main__": - sys.exit(_main()) diff --git a/.lib/git-fleximod/git_fleximod/submodule.py b/.lib/git-fleximod/git_fleximod/submodule.py deleted file mode 100644 index ba07c117..00000000 --- a/.lib/git-fleximod/git_fleximod/submodule.py +++ /dev/null @@ -1,459 +0,0 @@ -import os -import textwrap -import shutil -import string -from configparser import NoOptionError -from git_fleximod import utils -from git_fleximod.gitinterface import GitInterface - -class Submodule(): - """ - Represents a Git submodule with enhanced features for flexible management. - - Attributes: - name (str): The name of the submodule. - root_dir (str): The root directory of the main project. - path (str): The relative path from the root directory to the submodule. - url (str): The URL of the submodule repository. - fxurl (str): The URL for flexible submodule management (optional). - fxtag (str): The tag for flexible submodule management (optional). - fxsparse (str): Path to the sparse checkout file relative to the submodule path, see git-sparse-checkout for details (optional). - fxrequired (str): Indicates if the submodule is optional or required (optional). - logger (logging.Logger): Logger instance for logging (optional). - """ - def __init__(self, root_dir, name, path, url, fxtag=None, fxurl=None, fxsparse=None, fxrequired=None, logger=None): - """ - Initializes a new Submodule instance with the provided attributes. - """ - self.name = name - self.root_dir = root_dir - self.path = path - self.url = url - self.fxurl = fxurl - self.fxtag = fxtag - self.fxsparse = fxsparse - if fxrequired: - self.fxrequired = fxrequired - else: - self.fxrequired = "AlwaysRequired" - self.logger = logger - - def status(self): - """ - Checks the status of the submodule and returns 4 parameters: - - result (str): The status of the submodule. - - needsupdate (bool): An indicator if the submodule needs to be updated. - - localmods (bool): An indicator if the submodule has local modifications. - - testfails (bool): An indicator if the submodule has failed a test, this is used for testing purposes. - """ - - smpath = os.path.join(self.root_dir, self.path) - testfails = False - localmods = False - needsupdate = False - ahash = None - optional = "" - if "Optional" in self.fxrequired: - optional = " (optional)" - required = None - level = None - if not os.path.exists(os.path.join(smpath, ".git")): - rootgit = GitInterface(self.root_dir, self.logger) - # submodule commands use path, not name - status, tags = rootgit.git_operation("ls-remote", "--tags", self.url) - status, result = rootgit.git_operation("submodule","status",smpath) - result = result.split() - - if result: - ahash = result[0][1:] - hhash = None - atag = None - for htag in tags.split("\n"): - if htag.endswith('^{}'): - htag = htag[:-3] - if ahash and not atag and ahash in htag: - atag = (htag.split()[1])[10:] - if self.fxtag and not hhash and htag.endswith(self.fxtag): - hhash = htag.split()[0] - if hhash and atag: - break - if self.fxtag and (ahash == hhash or atag == self.fxtag): - result = f"e {self.name:>20} not checked out, aligned at tag {self.fxtag}{optional}" - needsupdate = True - elif self.fxtag: - status, ahash = rootgit.git_operation( - "submodule", "status", "{}".format(self.path) - ) - ahash = ahash[1 : len(self.fxtag) + 1] - if self.fxtag == ahash: - result = f"e {self.name:>20} not checked out, aligned at hash {ahash}{optional}" - else: - result = f"e {self.name:>20} not checked out, out of sync at tag {atag}, expected tag is {self.fxtag}{optional}" - testfails = True - needsupdate = True - else: - result = f"e {self.name:>20} has no fxtag defined in .gitmodules{optional}" - testfails = False - else: - with utils.pushd(smpath): - git = GitInterface(smpath, self.logger) - status, remote = git.git_operation("remote") - if remote == '': - result = f"e {self.name:>20} has no associated remote" - testfails = True - needsupdate = True - return result, needsupdate, localmods, testfails - status, rurl = git.git_operation("ls-remote","--get-url") - status, lines = git.git_operation("log", "--pretty=format:\"%h %d\"") - line = lines.partition('\n')[0] - parts = line.split() - ahash = parts[0][1:] - atag = None - if len(parts) > 3: - idx = 0 - while idx < len(parts)-1: - idx = idx+1 - if parts[idx] == 'tag:': - atag = parts[idx+1] - while atag.endswith(')') or atag.endswith(',') or atag.endswith("\""): - atag = atag[:-1] - if atag == self.fxtag: - break - recurse = False - if rurl != self.url: - remote = self._add_remote(git) - git.git_operation("fetch", remote) - # Asked for a tag and found that tag - if self.fxtag and atag == self.fxtag: - result = f" {self.name:>20} at tag {self.fxtag}" - recurse = True - testfails = False - # Asked for and found a hash - elif self.fxtag and (ahash[: len(self.fxtag)] == self.fxtag or (self.fxtag.find(ahash)==0)): - result = f" {self.name:>20} at hash {ahash}" - recurse = True - testfails = False - # Asked for and found a hash - elif atag == ahash: - result = f" {self.name:>20} at hash {ahash}" - recurse = True - # Did not find requested tag or hash - elif self.fxtag: - result = f"s {self.name:>20} {atag} {ahash} is out of sync with .gitmodules {self.fxtag}" - testfails = True - needsupdate = True - else: - if atag: - result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {atag}" - else: - result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {ahash}" - testfails = False - - status, output = git.git_operation("status", "--ignore-submodules", "-uno") - if "nothing to commit" not in output: - localmods = True - result = "M" + textwrap.indent(output, " ") -# print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}") - return result, needsupdate, localmods, testfails - - - def _add_remote(self, git): - """ - Adds a new remote to the submodule if it does not already exist. - - This method checks the existing remotes of the submodule. If the submodule's URL is not already listed as a remote, - it attempts to add a new remote. The name for the new remote is generated dynamically to avoid conflicts. If no - remotes exist, it defaults to naming the new remote 'origin'. - - Args: - git (GitInterface): An instance of GitInterface to perform git operations. - - Returns: - str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL. - """ - status, remotes = git.git_operation("remote", "-v") - remotes = remotes.splitlines() - upstream = None - if remotes: - status, upstream = git.git_operation("ls-remote", "--get-url") - newremote = "newremote.00" - tmpurl = self.url.replace("git@github.com:", "https://github.com/") - line = next((s for s in remotes if self.url in s or tmpurl in s), None) - if line: - newremote = line.split()[0] - return newremote - else: - i = 0 - while newremote in remotes: - i = i + 1 - newremote = f"newremote.{i:02d}" - else: - newremote = "origin" - git.git_operation("remote", "add", newremote, self.url) - return newremote - - def toplevel(self): - """ - Returns True if the submodule is Toplevel (either Required or Optional) - """ - return True if "Top" in self.fxrequired else False - - def sparse_checkout(self): - """ - Performs a sparse checkout of the submodule. - - This method optimizes the checkout process by only checking out files specified in the submodule's sparse-checkout configuration, - rather than the entire submodule content. It achieves this by first ensuring the `.git/info/sparse-checkout` file is created and - configured in the submodule's directory. Then, it proceeds to checkout the desired tag. If the submodule has already been checked out, - this method will not perform the checkout again. - - This approach is particularly beneficial for submodules with a large number of files, as it significantly reduces the time and disk space - required for the checkout process by avoiding the unnecessary checkout and subsequent removal of unneeded files. - - Returns: - None - """ - self.logger.info("Called sparse_checkout for {}".format(self.name)) - rgit = GitInterface(self.root_dir, self.logger) - status, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") - if superroot: - gitroot = superroot.strip() - else: - gitroot = self.root_dir - # Now need to move the .git dir to the submodule location - rootdotgit = os.path.join(self.root_dir, ".git") - while os.path.isfile(rootdotgit): - with open(rootdotgit) as f: - line = f.readline().rstrip() - if line.startswith("gitdir: "): - rootdotgit = os.path.abspath(os.path.join(self.root_dir,line[8:])) - assert os.path.isdir(rootdotgit) - # first create the module directory - if not os.path.isdir(os.path.join(self.root_dir, self.path)): - os.makedirs(os.path.join(self.root_dir, self.path)) - - # initialize a new git repo and set the sparse checkout flag - sprep_repo = os.path.join(self.root_dir, self.path) - sprepo_git = GitInterface(sprep_repo, self.logger) - if os.path.exists(os.path.join(sprep_repo, ".git")): - try: - self.logger.info("Submodule {} found".format(self.name)) - chk = sprepo_git.config_get_value("core", "sparseCheckout") - if chk == "true": - self.logger.info("Sparse submodule {} already checked out".format(self.name)) - return - except (NoOptionError): - self.logger.debug("Sparse submodule {} not present".format(self.name)) - except Exception as e: - utils.fatal_error("Unexpected error {} occured.".format(e)) - - sprepo_git.config_set_value("core", "sparseCheckout", "true") - - # set the repository remote - - self.logger.info("Setting remote origin in {}/{}".format(self.root_dir, self.path)) - status, remotes = sprepo_git.git_operation("remote", "-v") - if self.url not in remotes: - sprepo_git.git_operation("remote", "add", "origin", self.url) - - topgit = os.path.join(gitroot, ".git") - - if gitroot != self.root_dir and os.path.isfile(os.path.join(self.root_dir, ".git")): - with open(os.path.join(self.root_dir, ".git")) as f: - gitpath = os.path.relpath( - os.path.join(self.root_dir, f.read().split()[1]), - start=os.path.join(self.root_dir, self.path), - ) - rootdotgit = os.path.join(gitpath, "modules", self.name) - else: - rootdotgit = os.path.relpath( - os.path.join(self.root_dir, ".git", "modules", self.name), - start=os.path.join(self.root_dir, self.path), - ) - - if os.path.isdir(os.path.join(self.root_dir, self.path, ".git")): - with utils.pushd(sprep_repo): - if os.path.isdir(os.path.join(rootdotgit,".git")): - shutil.rmtree(os.path.join(rootdotgit,".git")) - shutil.move(".git", rootdotgit) - with open(".git", "w") as f: - f.write("gitdir: " + os.path.relpath(rootdotgit)) - infodir = os.path.join(rootdotgit, "info") - if not os.path.isdir(infodir): - os.makedirs(infodir) - gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout")) - if os.path.isfile(gitsparse): - self.logger.warning( - "submodule {} is already initialized {}".format(self.name, rootdotgit) - ) - os.remove(gitsparse) - - if os.path.isfile(self.fxsparse): - shutil.copy(self.fxsparse, gitsparse) - else: - self.logger.warning( - "submodule {} could not find {}".format(self.name, self.fxsparse) - ) - - # Finally checkout the repo - sprepo_git.git_operation("fetch", "origin", "--tags") - status,_ = sprepo_git.git_operation("checkout", self.fxtag) - if status: - print(f"Error checking out {self.name:>20} at {self.fxtag}") - else: - print(f"Successfully checked out {self.name:>20} at {self.fxtag}") - status,f = sprepo_git.git_operation("status") - # Restore any files deleted from sandbox - for line in f.splitlines(): - if "deleted:" in line: - deleted_file = line.split("deleted:")[1].strip() - sprepo_git.git_operation("checkout", deleted_file) - - rgit.config_set_value('submodule.' + self.name, "active", "true") - rgit.config_set_value('submodule.' + self.name, "url", self.url) - rgit.config_set_value('submodule.' + self.name, "path", self.path) - - async def update(self): - """ - Updates the submodule to the latest or specified version. - - This method handles the update process of the submodule, including checking out the submodule into the specified path, - handling sparse checkouts if configured, and updating the submodule's URL if necessary. It supports both SSH and HTTPS URLs, - automatically converting SSH URLs to HTTPS to avoid issues for users without SSH keys. - - The update process involves the following steps: - 1. If the submodule is configured for sparse checkout, it performs a sparse checkout. - 2. If the submodule is not already checked out, it clones the submodule using the provided URL. - 3. If a specific tag or hash is provided, it checks out that tag; otherwise, it checks out the latest version. - 4. If the root `.git` is a file (indicating a submodule or a worktree), additional steps are taken to integrate the submodule properly. - - Args: - None - Note: - - SSH URLs are automatically converted to HTTPS to accommodate users without SSH keys. - - Returns: - None - """ - git = GitInterface(self.root_dir, self.logger) - repodir = os.path.join(self.root_dir, self.path) - self.logger.info("Checkout {} into {}/{}".format(self.name, self.root_dir, self.path)) - # if url is provided update to the new url - tag = None - repo_exists = False - if os.path.exists(os.path.join(repodir, ".git")): - self.logger.info("Submodule {} already checked out".format(self.name)) - repo_exists = True - # Look for a .gitmodules file in the newly checkedout repo - if self.fxsparse: - print(f"Sparse checkout {self.name} fxsparse {self.fxsparse}") - if not os.path.isfile(self.fxsparse): - self.logger.info("Submodule {} fxsparse file not found".format(self.name)) - - self.sparse_checkout() - else: - if not repo_exists and self.url: - # ssh urls cause problems for those who dont have git accounts with ssh keys defined - # but cime has one since e3sm prefers ssh to https, because the .gitmodules file was - # opened with a GitModules object we don't need to worry about restoring the file here - # it will be done by the GitModules class - if self.url.startswith("git@"): - git.git_operation("clone", self.url, self.path) - smgit = GitInterface(repodir, self.logger) - if not tag: - status, tag = smgit.git_operation("describe", "--tags", "--always") - smgit.git_operation("checkout", tag) - # Now need to move the .git dir to the submodule location - rootdotgit = os.path.join(self.root_dir, ".git") - if os.path.isfile(rootdotgit): - with open(rootdotgit) as f: - line = f.readline() - if line.startswith("gitdir: "): - rootdotgit = line[8:] - - newpath = os.path.abspath(os.path.join(self.root_dir, rootdotgit, "modules", self.name)) - if os.path.exists(newpath): - shutil.rmtree(os.path.join(repodir, ".git")) - else: - shutil.move(os.path.join(repodir, ".git"), newpath) - - with open(os.path.join(repodir, ".git"), "w") as f: - f.write("gitdir: " + os.path.relpath(newpath, start=repodir)) - - if not os.path.exists(repodir): - parent = os.path.dirname(repodir) - if not os.path.isdir(parent): - os.makedirs(parent) - git.git_operation("submodule", "add", "--name", self.name, "--", self.url, self.path) - - if not repo_exists: - git.git_operation("submodule", "init", "--", self.path) - await git.git_operation_async("submodule", "update", "--", self.path) - - if self.fxtag: - smgit = GitInterface(repodir, self.logger) - newremote = self._add_remote(smgit) - # Trying to distingush a tag from a hash - allowed = set(string.digits + 'abcdef') - status = 0 - if not set(self.fxtag) <= allowed: - # This is a tag - tag = f"refs/tags/{self.fxtag}:refs/tags/{self.fxtag}" - status,_ = smgit.git_operation("fetch", newremote, tag) - if status == 0: - status,_ = smgit.git_operation("checkout", self.fxtag) - if status: - utils.fatal_error( - f"Failed to checkout {self.name} at tag or hash {self.fxtag} from {repodir}" - ) - - if not os.path.exists(os.path.join(repodir, ".git")): - utils.fatal_error( - f"Failed to checkout {self.name} {repo_exists} {repodir} {self.path}" - ) - - - if os.path.exists(os.path.join(self.path, ".git")): - submoddir = os.path.join(self.root_dir, self.path) - with utils.pushd(submoddir): - git = GitInterface(submoddir, self.logger) - # first make sure the url is correct - newremote = self._add_remote(git) - status, tags = git.git_operation("tag", "-l") - fxtag = self.fxtag - if fxtag and fxtag not in tags: - git.git_operation("fetch", newremote, "--tags") - status, atag = git.git_operation("describe", "--tags", "--always") - status, files = git.git_operation("diff", "--name-only", "-z") - modfiles = [] - moddirs = [] - if files: - for f in files.split('\0'): - if f: - if os.path.exists(f): - git.git_operation("checkout",f) - elif os.path.isdir(f): - moddirs.append(f) - else: - modfiles.append(f) - if fxtag and fxtag != atag: - try: - status, _ = git.git_operation("checkout", fxtag) - if not status: - print(f"{self.name:>20} updated to {fxtag}") - except Exception as error: - print(error) - - - elif not fxtag: - print(f"No fxtag found for submodule {self.name:>20}") - elif modfiles: - print(f"{self.name:>20} has modified files: {modfiles}") - elif moddirs: - print(f"{self.name:>20} has modified directories: {moddirs}") - else: - print(f"{self.name:>20} up to date.") - - - - return diff --git a/.lib/git-fleximod/git_fleximod/utils.py b/.lib/git-fleximod/git_fleximod/utils.py deleted file mode 100644 index c4f43d52..00000000 --- a/.lib/git-fleximod/git_fleximod/utils.py +++ /dev/null @@ -1,365 +0,0 @@ -#!/usr/bin/env python3 -""" -Common public utilities for manic package - -""" - -import logging -import os -import subprocess -import sys -from threading import Timer -from pathlib import Path - -LOCAL_PATH_INDICATOR = "." -# --------------------------------------------------------------------- -# -# functions to massage text for output and other useful utilities -# -# --------------------------------------------------------------------- -from contextlib import contextmanager - - -@contextmanager -def pushd(new_dir): - """context for chdir. usage: with pushd(new_dir)""" - previous_dir = os.getcwd() - os.chdir(new_dir) - try: - yield - finally: - os.chdir(previous_dir) - - -def log_process_output(output): - """Log each line of process output at debug level so it can be - filtered if necessary. By default, output is a single string, and - logging.debug(output) will only put log info heading on the first - line. This makes it hard to filter with grep. - - """ - output = output.split("\n") - for line in output: - logging.debug(line) - - -def printlog(msg, **kwargs): - """Wrapper script around print to ensure that everything printed to - the screen also gets logged. - - """ - logging.info(msg) - if kwargs: - print(msg, **kwargs) - else: - print(msg) - sys.stdout.flush() - - -def find_upwards(root_dir, filename): - """Find a file in root dir or any of it's parents""" - d = Path(root_dir) - root = Path(d.root) - while d != root: - attempt = d / filename - if attempt.exists(): - return attempt - d = d.parent - return None - - -def last_n_lines(the_string, n_lines, truncation_message=None): - """Returns the last n lines of the given string - - Args: - the_string: str - n_lines: int - truncation_message: str, optional - - Returns a string containing the last n lines of the_string - - If truncation_message is provided, the returned string begins with - the given message if and only if the string is greater than n lines - to begin with. - """ - - lines = the_string.splitlines(True) - if len(lines) <= n_lines: - return_val = the_string - else: - lines_subset = lines[-n_lines:] - str_truncated = "".join(lines_subset) - if truncation_message: - str_truncated = truncation_message + "\n" + str_truncated - return_val = str_truncated - - return return_val - - -def indent_string(the_string, indent_level): - """Indents the given string by a given number of spaces - - Args: - the_string: str - indent_level: int - - Returns a new string that is the same as the_string, except that - each line is indented by 'indent_level' spaces. - - In python3, this can be done with textwrap.indent. - """ - - lines = the_string.splitlines(True) - padding = " " * indent_level - lines_indented = [padding + line for line in lines] - return "".join(lines_indented) - - -# --------------------------------------------------------------------- -# -# error handling -# -# --------------------------------------------------------------------- - - -def fatal_error(message): - """ - Error output function - """ - logging.error(message) - raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message)) - - -# --------------------------------------------------------------------- -# -# Data conversion / manipulation -# -# --------------------------------------------------------------------- -def str_to_bool(bool_str): - """Convert a sting representation of as boolean into a true boolean. - - Conversion should be case insensitive. - """ - value = None - str_lower = bool_str.lower() - if str_lower in ("true", "t"): - value = True - elif str_lower in ("false", "f"): - value = False - if value is None: - msg = ( - 'ERROR: invalid boolean string value "{0}". ' - 'Must be "true" or "false"'.format(bool_str) - ) - fatal_error(msg) - return value - - -REMOTE_PREFIXES = ["http://", "https://", "ssh://", "git@"] - - -def is_remote_url(url): - """check if the user provided a local file path instead of a - remote. If so, it must be expanded to an absolute - path. - - """ - remote_url = False - for prefix in REMOTE_PREFIXES: - if url.startswith(prefix): - remote_url = True - return remote_url - - -def split_remote_url(url): - """check if the user provided a local file path or a - remote. If remote, try to strip off protocol info. - - """ - remote_url = is_remote_url(url) - if not remote_url: - return url - - for prefix in REMOTE_PREFIXES: - url = url.replace(prefix, "") - - if "@" in url: - url = url.split("@")[1] - - if ":" in url: - url = url.split(":")[1] - - return url - - -def expand_local_url(url, field): - """check if the user provided a local file path instead of a - remote. If so, it must be expanded to an absolute - path. - - Note: local paths of LOCAL_PATH_INDICATOR have special meaning and - represent local copy only, don't work with the remotes. - - """ - remote_url = is_remote_url(url) - if not remote_url: - if url.strip() == LOCAL_PATH_INDICATOR: - pass - else: - url = os.path.expandvars(url) - url = os.path.expanduser(url) - if not os.path.isabs(url): - msg = ( - 'WARNING: Externals description for "{0}" contains a ' - "url that is not remote and does not expand to an " - "absolute path. Version control operations may " - "fail.\n\nurl={1}".format(field, url) - ) - printlog(msg) - else: - url = os.path.normpath(url) - return url - - -# --------------------------------------------------------------------- -# -# subprocess -# -# --------------------------------------------------------------------- - -# Give the user a helpful message if we detect that a command seems to -# be hanging. -_HANGING_SEC = 300 - - -def _hanging_msg(working_directory, command): - print( - """ - -Command '{command}' -from directory {working_directory} -has taken {hanging_sec} seconds. It may be hanging. - -The command will continue to run, but you may want to abort -git-fleximod with ^C and investigate. A possible cause of hangs is git -requires authentication to access a private repository. On some -systems, git requests for authentication information will not -be displayed to the user. In this case, the program will appear to -hang. Ensure you can run git manually and access all -repositories without entering your authentication information. - -""".format( - command=command, - working_directory=working_directory, - hanging_sec=_HANGING_SEC, - ) - ) - - -def execute_subprocess(commands, status_to_caller=False, output_to_caller=False): - """Wrapper around subprocess.check_output to handle common - exceptions. - - check_output runs a command with arguments and waits - for it to complete. - - check_output raises an exception on a nonzero return code. if - status_to_caller is true, execute_subprocess returns the subprocess - return code, otherwise execute_subprocess treats non-zero return - status as an error and raises an exception. - - """ - cwd = os.getcwd() - msg = "In directory: {0}\nexecute_subprocess running command:".format(cwd) - logging.info(msg) - commands_str = " ".join(str(element) for element in commands) - logging.info(commands_str) - return_to_caller = status_to_caller or output_to_caller - status = -1 - output = "" - hanging_timer = Timer( - _HANGING_SEC, - _hanging_msg, - kwargs={"working_directory": cwd, "command": commands_str}, - ) - hanging_timer.start() - try: - output = subprocess.check_output( - commands, stderr=subprocess.STDOUT, universal_newlines=True - ) - log_process_output(output) - status = 0 - except OSError as error: - msg = failed_command_msg( - "Command execution failed. Does the executable exist?", commands - ) - logging.error(error) - fatal_error(msg) - except ValueError as error: - msg = failed_command_msg( - "DEV_ERROR: Invalid arguments trying to run subprocess", commands - ) - logging.error(error) - fatal_error(msg) - except subprocess.CalledProcessError as error: - # Only report the error if we are NOT returning to the - # caller. If we are returning to the caller, then it may be a - # simple status check. If returning, it is the callers - # responsibility determine if an error occurred and handle it - # appropriately. - msg_context = ( - "Process did not run successfully; " - "returned status {0}".format(error.returncode) - ) - msg = failed_command_msg(msg_context, commands, output=error.output) - if not return_to_caller: - logging.error(error) - logging.error(msg) - log_process_output(error.output) - fatal_error(msg) - status = error.returncode - finally: - hanging_timer.cancel() - - if status_to_caller and output_to_caller: - ret_value = (status, output) - elif status_to_caller: - ret_value = status - elif output_to_caller: - ret_value = output - else: - ret_value = None - - return ret_value - - -def failed_command_msg(msg_context, command, output=None): - """Template for consistent error messages from subprocess calls. - - If 'output' is given, it should provide the output from the failed - command - """ - - if output: - output_truncated = last_n_lines( - output, 20, truncation_message="[... Output truncated for brevity ...]" - ) - errmsg = ( - "Failed with output:\n" + indent_string(output_truncated, 4) + "\nERROR: " - ) - else: - errmsg = "" - - command_str = " ".join(command) - errmsg += """In directory - {cwd} -{context}: - {command} -""".format( - cwd=os.getcwd(), context=msg_context, command=command_str - ) - - if output: - errmsg += "See above for output from failed command.\n" - - return errmsg diff --git a/.lib/git-fleximod/poetry.lock b/.lib/git-fleximod/poetry.lock deleted file mode 100644 index ac82fb0d..00000000 --- a/.lib/git-fleximod/poetry.lock +++ /dev/null @@ -1,693 +0,0 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. - -[[package]] -name = "alabaster" -version = "0.7.13" -description = "A configurable sidebar-enabled Sphinx theme" -optional = false -python-versions = ">=3.6" -files = [ - {file = "alabaster-0.7.13-py3-none-any.whl", hash = "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3"}, - {file = "alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"}, -] - -[[package]] -name = "babel" -version = "2.15.0" -description = "Internationalization utilities" -optional = false -python-versions = ">=3.8" -files = [ - {file = "Babel-2.15.0-py3-none-any.whl", hash = "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb"}, - {file = "babel-2.15.0.tar.gz", hash = "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"}, -] - -[package.dependencies] -pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} - -[package.extras] -dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] - -[[package]] -name = "certifi" -version = "2024.8.30" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.6" -files = [ - {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, - {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.3.2" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, - {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, -] - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "docutils" -version = "0.19" -description = "Docutils -- Python Documentation Utilities" -optional = false -python-versions = ">=3.7" -files = [ - {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, - {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, -] - -[[package]] -name = "exceptiongroup" -version = "1.2.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, - {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, -] - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "fsspec" -version = "2023.12.2" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -files = [ - {file = "fsspec-2023.12.2-py3-none-any.whl", hash = "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"}, - {file = "fsspec-2023.12.2.tar.gz", hash = "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb"}, -] - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -devel = ["pytest", "pytest-cov"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -tqdm = ["tqdm"] - -[[package]] -name = "gitdb" -version = "4.0.11" -description = "Git Object Database" -optional = false -python-versions = ">=3.7" -files = [ - {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, - {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, -] - -[package.dependencies] -smmap = ">=3.0.1,<6" - -[[package]] -name = "gitpython" -version = "3.1.43" -description = "GitPython is a Python library used to interact with Git repositories" -optional = false -python-versions = ">=3.7" -files = [ - {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, - {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, -] - -[package.dependencies] -gitdb = ">=4.0.1,<5" - -[package.extras] -doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] - -[[package]] -name = "idna" -version = "3.7" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.5" -files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, -] - -[[package]] -name = "imagesize" -version = "1.4.1" -description = "Getting image size from png/jpeg/jpeg2000/gif file" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, - {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, -] - -[[package]] -name = "importlib-metadata" -version = "8.0.0" -description = "Read metadata from Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "importlib_metadata-8.0.0-py3-none-any.whl", hash = "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f"}, - {file = "importlib_metadata-8.0.0.tar.gz", hash = "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812"}, -] - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -perf = ["ipython"] -test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] - -[[package]] -name = "iniconfig" -version = "2.0.0" -description = "brain-dead simple config-ini parsing" -optional = false -python-versions = ">=3.7" -files = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, -] - -[[package]] -name = "jinja2" -version = "3.1.4" -description = "A very fast and expressive template engine." -optional = false -python-versions = ">=3.7" -files = [ - {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, - {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, -] - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "markupsafe" -version = "2.1.5" -description = "Safely add untrusted strings to HTML/XML markup." -optional = false -python-versions = ">=3.7" -files = [ - {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, - {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, -] - -[[package]] -name = "packaging" -version = "24.1" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, - {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, -] - -[[package]] -name = "pluggy" -version = "1.5.0" -description = "plugin and hook calling mechanisms for python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, - {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, -] - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "pyfakefs" -version = "5.5.0" -description = "pyfakefs implements a fake file system that mocks the Python file system modules." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pyfakefs-5.5.0-py3-none-any.whl", hash = "sha256:8dbf203ab7bef1529f11f7d41b9478b898e95bf9f3b71262163aac07a518cd76"}, - {file = "pyfakefs-5.5.0.tar.gz", hash = "sha256:7448aaa07142f892d0a4eb52a5ed3206a9f02c6599e686cd97d624c18979c154"}, -] - -[[package]] -name = "pygments" -version = "2.18.0" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.8" -files = [ - {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, - {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, -] - -[package.extras] -windows-terminal = ["colorama (>=0.4.6)"] - -[[package]] -name = "pytest" -version = "8.2.2" -description = "pytest: simple powerful testing with Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, - {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=1.5,<2.0" -tomli = {version = ">=1", markers = "python_version < \"3.11\""} - -[package.extras] -dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] - -[[package]] -name = "pytz" -version = "2024.1" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -files = [ - {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, - {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, -] - -[[package]] -name = "requests" -version = "2.32.3" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.8" -files = [ - {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, - {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, -] - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "smmap" -version = "5.0.1" -description = "A pure Python implementation of a sliding window memory map manager" -optional = false -python-versions = ">=3.7" -files = [ - {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, - {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, -] - -[[package]] -name = "snowballstemmer" -version = "2.2.0" -description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." -optional = false -python-versions = "*" -files = [ - {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, - {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, -] - -[[package]] -name = "sphinx" -version = "5.3.0" -description = "Python documentation generator" -optional = false -python-versions = ">=3.6" -files = [ - {file = "Sphinx-5.3.0.tar.gz", hash = "sha256:51026de0a9ff9fc13c05d74913ad66047e104f56a129ff73e174eb5c3ee794b5"}, - {file = "sphinx-5.3.0-py3-none-any.whl", hash = "sha256:060ca5c9f7ba57a08a1219e547b269fadf125ae25b06b9fa7f66768efb652d6d"}, -] - -[package.dependencies] -alabaster = ">=0.7,<0.8" -babel = ">=2.9" -colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} -docutils = ">=0.14,<0.20" -imagesize = ">=1.3" -importlib-metadata = {version = ">=4.8", markers = "python_version < \"3.10\""} -Jinja2 = ">=3.0" -packaging = ">=21.0" -Pygments = ">=2.12" -requests = ">=2.5.0" -snowballstemmer = ">=2.0" -sphinxcontrib-applehelp = "*" -sphinxcontrib-devhelp = "*" -sphinxcontrib-htmlhelp = ">=2.0.0" -sphinxcontrib-jsmath = "*" -sphinxcontrib-qthelp = "*" -sphinxcontrib-serializinghtml = ">=1.1.5" - -[package.extras] -docs = ["sphinxcontrib-websupport"] -lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-bugbear", "flake8-comprehensions", "flake8-simplify", "isort", "mypy (>=0.981)", "sphinx-lint", "types-requests", "types-typed-ast"] -test = ["cython", "html5lib", "pytest (>=4.6)", "typed_ast"] - -[[package]] -name = "sphinxcontrib-applehelp" -version = "1.0.4" -description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" -optional = false -python-versions = ">=3.8" -files = [ - {file = "sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e"}, - {file = "sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["pytest"] - -[[package]] -name = "sphinxcontrib-devhelp" -version = "1.0.2" -description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." -optional = false -python-versions = ">=3.5" -files = [ - {file = "sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"}, - {file = "sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["pytest"] - -[[package]] -name = "sphinxcontrib-htmlhelp" -version = "2.0.1" -description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" -optional = false -python-versions = ">=3.8" -files = [ - {file = "sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff"}, - {file = "sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["html5lib", "pytest"] - -[[package]] -name = "sphinxcontrib-jsmath" -version = "1.0.1" -description = "A sphinx extension which renders display math in HTML via JavaScript" -optional = false -python-versions = ">=3.5" -files = [ - {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, - {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, -] - -[package.extras] -test = ["flake8", "mypy", "pytest"] - -[[package]] -name = "sphinxcontrib-qthelp" -version = "1.0.3" -description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." -optional = false -python-versions = ">=3.5" -files = [ - {file = "sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72"}, - {file = "sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["pytest"] - -[[package]] -name = "sphinxcontrib-serializinghtml" -version = "1.1.5" -description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." -optional = false -python-versions = ">=3.5" -files = [ - {file = "sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952"}, - {file = "sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["pytest"] - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] - -[[package]] -name = "urllib3" -version = "2.2.2" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.8" -files = [ - {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, - {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, -] - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "wheel" -version = "0.42.0" -description = "A built-package format for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "wheel-0.42.0-py3-none-any.whl", hash = "sha256:177f9c9b0d45c47873b619f5b650346d632cdc35fb5e4d25058e09c9e581433d"}, - {file = "wheel-0.42.0.tar.gz", hash = "sha256:c45be39f7882c9d34243236f2d63cbd58039e360f85d0913425fbd7ceea617a8"}, -] - -[package.extras] -test = ["pytest (>=6.0.0)", "setuptools (>=65)"] - -[[package]] -name = "zipp" -version = "3.19.2" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.8" -files = [ - {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, - {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, -] - -[package.extras] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] - -[metadata] -lock-version = "2.0" -python-versions = "^3.8" -content-hash = "25ee2ae1d74abedde3a6637a60d4a3095ea5cf9731960875741bbc2ba84a475d" diff --git a/.lib/git-fleximod/pyproject.toml b/.lib/git-fleximod/pyproject.toml deleted file mode 100644 index 029fd65e..00000000 --- a/.lib/git-fleximod/pyproject.toml +++ /dev/null @@ -1,42 +0,0 @@ -[tool.poetry] -name = "git-fleximod" -version = "1.0.2" -description = "Extended support for git-submodule and git-sparse-checkout" -authors = ["Jim Edwards "] -maintainers = ["Jim Edwards "] -license = "MIT" -readme = "README.md" -homepage = "https://github.com/jedwards4b/git-fleximod" -keywords = ["git", "submodule", "sparse-checkout"] -packages = [ -{ include = "git_fleximod"}, -{ include = "doc"}, -{ include = "README.md"}, -] - -[tool.poetry.scripts] -git-fleximod = "git_fleximod.git_fleximod:main" -me2flexi = "git_fleximod.metoflexi:_main" -fsspec = "fsspec.fuse:main" - -[tool.poetry.dependencies] -python = "^3.8" -GitPython = "^3.1.0" -sphinx = "^5.0.0" -fsspec = "^2023.12.2" -wheel = "^0.42.0" -pytest = "^8.0.0" -pyfakefs = "^5.3.5" - -[tool.poetry.urls] -"Bug Tracker" = "https://github.com/jedwards4b/git-fleximod/issues" - -[tool.pytest.ini_options] -markers = [ - "skip_after_first: only run on first iteration" -] - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" - diff --git a/.lib/git-fleximod/tbump.toml b/.lib/git-fleximod/tbump.toml deleted file mode 100644 index f6fe1e88..00000000 --- a/.lib/git-fleximod/tbump.toml +++ /dev/null @@ -1,43 +0,0 @@ -# Uncomment this if your project is hosted on GitHub: -github_url = "https://github.com/jedwards4b/git-fleximod/" - -[version] -current = "1.0.2" - -# Example of a semver regexp. -# Make sure this matches current_version before -# using tbump -regex = ''' - (?P\d+) - \. - (?P\d+) - \. - (?P\d+) - ''' - -[git] -message_template = "Bump to {new_version}" -tag_template = "v{new_version}" - -# For each file to patch, add a [[file]] config -# section containing the path of the file, relative to the -# tbump.toml location. -[[file]] -src = "git_fleximod/cli.py" - -[[file]] -src = "pyproject.toml" - -# You can specify a list of commands to -# run after the files have been patched -# and before the git commit is made - -# [[before_commit]] -# name = "check changelog" -# cmd = "grep -q {new_version} Changelog.rst" - -# Or run some commands after the git tag and the branch -# have been pushed: -# [[after_push]] -# name = "publish" -# cmd = "./publish.sh" diff --git a/.lib/git-fleximod/tests/__init__.py b/.lib/git-fleximod/tests/__init__.py deleted file mode 100644 index 4d4c66c7..00000000 --- a/.lib/git-fleximod/tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -import sys, os - -sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir, "src")) diff --git a/.lib/git-fleximod/tests/conftest.py b/.lib/git-fleximod/tests/conftest.py deleted file mode 100644 index 44d28e17..00000000 --- a/.lib/git-fleximod/tests/conftest.py +++ /dev/null @@ -1,150 +0,0 @@ -import pytest -from git_fleximod.gitinterface import GitInterface -import os -import subprocess -import logging -from pathlib import Path - -@pytest.fixture(scope='session') -def logger(): - logging.basicConfig( - level=logging.INFO, format="%(name)s - %(levelname)s - %(message)s", handlers=[logging.StreamHandler()] - ) - logger = logging.getLogger(__name__) - return logger - -all_repos=[ - {"subrepo_path": "modules/test", - "submodule_name": "test_submodule", - "status1" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", - "status2" : "test_submodule at tag MPIserial_2.4.0", - "status3" : "test_submodule at tag MPIserial_2.4.0", - "status4" : "test_submodule at tag MPIserial_2.4.0", - "gitmodules_content" : """ - [submodule "test_submodule"] - path = modules/test - url = https://github.com/ESMCI/mpi-serial.git - fxtag = MPIserial_2.4.0 - fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = ToplevelRequired -"""}, - {"subrepo_path": "modules/test_optional", - "submodule_name": "test_optional", - "status1" : "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", - "status2" : "test_optional at tag MPIserial_2.4.0", - "status3" : "test_optional not checked out, out of sync at tag MPIserial_2.5.4, expected tag is MPIserial_2.4.0 (optional)", - "status4" : "test_optional at tag MPIserial_2.4.0", - "gitmodules_content": """ - [submodule "test_optional"] - path = modules/test_optional - url = https://github.com/ESMCI/mpi-serial.git - fxtag = MPIserial_2.4.0 - fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = ToplevelOptional -"""}, - {"subrepo_path": "modules/test_alwaysoptional", - "submodule_name": "test_alwaysoptional", - "status1" : "test_alwaysoptional MPIserial_2.3.0 is out of sync with .gitmodules e5cf35c", - "status2" : "test_alwaysoptional at hash e5cf35c", - "status3" : "out of sync at tag MPIserial_2.5.4, expected tag is e5cf35c", - "status4" : "test_alwaysoptional at hash e5cf35c", - "gitmodules_content": """ - [submodule "test_alwaysoptional"] - path = modules/test_alwaysoptional - url = https://github.com/ESMCI/mpi-serial.git - fxtag = e5cf35c - fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = AlwaysOptional -"""}, - {"subrepo_path": "modules/test_sparse", - "submodule_name": "test_sparse", - "status1" : "test_sparse at tag MPIserial_2.5.0", - "status2" : "test_sparse at tag MPIserial_2.5.0", - "status3" : "test_sparse at tag MPIserial_2.5.0", - "status4" : "test_sparse at tag MPIserial_2.5.0", - "gitmodules_content": """ - [submodule "test_sparse"] - path = modules/test_sparse - url = https://github.com/ESMCI/mpi-serial.git - fxtag = MPIserial_2.5.0 - fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = AlwaysRequired - fxsparse = ../.sparse_file_list -"""}, -] -@pytest.fixture(params=all_repos) - -def shared_repos(request): - return request.param - -@pytest.fixture -def get_all_repos(): - return all_repos - -def write_sparse_checkout_file(fp): - sparse_content = """m4 -""" - fp.write_text(sparse_content) - -@pytest.fixture -def test_repo(shared_repos, tmp_path, logger): - subrepo_path = shared_repos["subrepo_path"] - submodule_name = shared_repos["submodule_name"] - test_dir = tmp_path / "testrepo" - test_dir.mkdir() - str_path = str(test_dir) - gitp = GitInterface(str_path, logger) - assert test_dir.joinpath(".git").is_dir() - (test_dir / "modules").mkdir() - if "sparse" in submodule_name: - (test_dir / subrepo_path).mkdir() - # Add the sparse checkout file - write_sparse_checkout_file(test_dir / "modules" / ".sparse_file_list") - gitp.git_operation("add","modules/.sparse_file_list") - else: - gitp = GitInterface(str(test_dir), logger) - gitp.git_operation("submodule", "add", "--depth","1","--name", submodule_name, "https://github.com/ESMCI/mpi-serial.git", subrepo_path) - assert test_dir.joinpath(".gitmodules").is_file() - gitp.git_operation("add",subrepo_path) - gitp.git_operation("commit","-a","-m","\"add submod\"") - test_dir2 = tmp_path / "testrepo2" - gitp.git_operation("clone",test_dir,test_dir2) - return test_dir2 - - -@pytest.fixture -def complex_repo(tmp_path, logger): - test_dir = tmp_path / "testcomplex" - test_dir.mkdir() - str_path = str(test_dir) - gitp = GitInterface(str_path, logger) - gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") - gitp.git_operation("fetch", "origin") - gitp.git_operation("checkout", "v0.0.1") - return test_dir - -@pytest.fixture -def complex_update(tmp_path, logger): - test_dir = tmp_path / "testcomplex" - test_dir.mkdir() - str_path = str(test_dir) - gitp = GitInterface(str_path, logger) - gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") - gitp.git_operation("fetch", "origin") - gitp.git_operation("checkout", "v0.0.2") - - return test_dir - -@pytest.fixture -def git_fleximod(): - def _run_fleximod(path, args, input=None): - cmd = ["git", "fleximod"] + args.split() - result = subprocess.run(cmd, cwd=path, input=input, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True) - if result.returncode: - print(result.stdout) - print(result.stderr) - return result - return _run_fleximod - diff --git a/.lib/git-fleximod/tests/test_a_import.py b/.lib/git-fleximod/tests/test_a_import.py deleted file mode 100644 index d5ca878d..00000000 --- a/.lib/git-fleximod/tests/test_a_import.py +++ /dev/null @@ -1,8 +0,0 @@ -# pylint: disable=unused-import -from git_fleximod import cli -from git_fleximod import utils -from git_fleximod.gitinterface import GitInterface -from git_fleximod.gitmodules import GitModules - -def test_import(): - print("here") diff --git a/.lib/git-fleximod/tests/test_b_update.py b/.lib/git-fleximod/tests/test_b_update.py deleted file mode 100644 index 159f1cfa..00000000 --- a/.lib/git-fleximod/tests/test_b_update.py +++ /dev/null @@ -1,26 +0,0 @@ -import pytest -from pathlib import Path - -def test_basic_checkout(git_fleximod, test_repo, shared_repos): - # Prepare a simple .gitmodules - gm = shared_repos['gitmodules_content'] - file_path = (test_repo / ".gitmodules") - repo_name = shared_repos["submodule_name"] - repo_path = shared_repos["subrepo_path"] - - file_path.write_text(gm) - - # Run the command - result = git_fleximod(test_repo, f"update {repo_name}") - - # Assertions - assert result.returncode == 0 - assert Path(test_repo / repo_path).exists() # Did the submodule directory get created? - if "sparse" in repo_name: - assert Path(test_repo / f"{repo_path}/m4").exists() # Did the submodule sparse directory get created? - assert not Path(test_repo / f"{repo_path}/README").exists() # Did only the submodule sparse directory get created? - - status = git_fleximod(test_repo, f"status {repo_name}") - - assert shared_repos["status2"] in status.stdout - diff --git a/.lib/git-fleximod/tests/test_c_required.py b/.lib/git-fleximod/tests/test_c_required.py deleted file mode 100644 index 2ac66145..00000000 --- a/.lib/git-fleximod/tests/test_c_required.py +++ /dev/null @@ -1,43 +0,0 @@ -import pytest -import re -from pathlib import Path - -def test_required(git_fleximod, test_repo, shared_repos): - file_path = (test_repo / ".gitmodules") - gm = shared_repos["gitmodules_content"] - repo_name = shared_repos["submodule_name"] - if file_path.exists(): - with file_path.open("r") as f: - gitmodules_content = f.read() - # add the entry if it does not exist - if repo_name not in gitmodules_content: - file_path.write_text(gitmodules_content+gm) - # or if it is incomplete - elif gm not in gitmodules_content: - file_path.write_text(gm) - else: - file_path.write_text(gm) - result = git_fleximod(test_repo, "update") - assert result.returncode == 0 - status = git_fleximod(test_repo, f"status {repo_name}") - assert shared_repos["status3"] in status.stdout - status = git_fleximod(test_repo, f"update --optional") - assert result.returncode == 0 - status = git_fleximod(test_repo, f"status {repo_name}") - assert shared_repos["status4"] in status.stdout - status = git_fleximod(test_repo, f"update {repo_name}") - assert result.returncode == 0 - status = git_fleximod(test_repo, f"status {repo_name}") - assert shared_repos["status4"] in status.stdout - - text = file_path.read_text() - new_value = "somethingelse" - pattern = r"(^\s*fxtag\s*=\s*).*$" - replacement = r"\1" + new_value - new_text = re.sub(pattern, replacement, text, flags=re.MULTILINE) - - # Write updated content back to file - file_path.write_text(new_text) - - result = git_fleximod(test_repo, f"update {repo_name}") - assert f'fatal: couldn\'t find remote ref' in result.stderr or 'error: pathspec \'somethingelse\' did not match any file(s) known to git' in result.stderr diff --git a/.lib/git-fleximod/tests/test_d_complex.py b/.lib/git-fleximod/tests/test_d_complex.py deleted file mode 100644 index edde7d81..00000000 --- a/.lib/git-fleximod/tests/test_d_complex.py +++ /dev/null @@ -1,66 +0,0 @@ -import pytest -from pathlib import Path -from git_fleximod.gitinterface import GitInterface - -def test_complex_checkout(git_fleximod, complex_repo, logger): - status = git_fleximod(complex_repo, "status") - assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) - assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout) - assert("Complex not checked out, aligned at tag testtag02" in status.stdout) - assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) - - # This should checkout and update test_submodule and complex_sub - result = git_fleximod(complex_repo, "update") - assert result.returncode == 0 - - status = git_fleximod(complex_repo, "status") - assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag02" in status.stdout) - - # now check the complex_sub - root = (complex_repo / "modules" / "complex") - assert(not (root / "libraries" / "gptl" / ".git").exists()) - assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serial" / ".git").exists()) - assert(not (root / "modules" / "mpi-serial2" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / "m4").exists()) - assert(not (root / "modules" / "mpi-sparse" / "README").exists()) - - # update a single optional submodule - - result = git_fleximod(complex_repo, "update ToplevelOptional") - assert result.returncode == 0 - - status = git_fleximod(complex_repo, "status") - assert("ToplevelOptional at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag02" in status.stdout) - assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) - - # Finally update optional - result = git_fleximod(complex_repo, "update --optional") - assert result.returncode == 0 - - status = git_fleximod(complex_repo, "status") - assert("ToplevelOptional at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag02" in status.stdout) - assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout) - - # now check the complex_sub - root = (complex_repo / "modules" / "complex" ) - assert(not (root / "libraries" / "gptl" / ".git").exists()) - assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serial2" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / "m4").exists()) - assert(not (root / "modules" / "mpi-sparse" / "README").exists()) - - diff --git a/.lib/git-fleximod/tests/test_e_complex_update.py b/.lib/git-fleximod/tests/test_e_complex_update.py deleted file mode 100644 index 0c3ab4c6..00000000 --- a/.lib/git-fleximod/tests/test_e_complex_update.py +++ /dev/null @@ -1,69 +0,0 @@ -import pytest -from pathlib import Path -from git_fleximod.gitinterface import GitInterface - -def test_complex_update(git_fleximod, complex_update, logger): - status = git_fleximod(complex_update, "status") - assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) - assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout) - assert("Complex not checked out, out of sync at tag testtag02, expected tag is testtag3" in status.stdout) - assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) - - # This should checkout and update test_submodule and complex_sub - result = git_fleximod(complex_update, "update") - assert result.returncode == 0 - - status = git_fleximod(complex_update, "status") - assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag3" in status.stdout) - - # now check the complex_sub - root = (complex_update / "modules" / "complex") - assert(not (root / "libraries" / "gptl" / ".git").exists()) - assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serialAR" / ".git").exists()) - assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) - assert(not (root / "modules" / "mpi-serial2" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / "m4").exists()) - assert(not (root / "modules" / "mpi-sparse" / "README").exists()) - - # update a single optional submodule - - result = git_fleximod(complex_update, "update ToplevelOptional") - assert result.returncode == 0 - - status = git_fleximod(complex_update, "status") - assert("ToplevelOptional at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag3" in status.stdout) - assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) - - # Finally update optional - result = git_fleximod(complex_update, "update --optional") - assert result.returncode == 0 - - status = git_fleximod(complex_update, "status") - assert("ToplevelOptional at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag3" in status.stdout) - assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout) - - # now check the complex_sub - root = (complex_update / "modules" / "complex" ) - assert(not (root / "libraries" / "gptl" / ".git").exists()) - assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) - assert(not (root / "modules" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serialAR" / ".git").exists()) - assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / ".git").exists()) - assert((root / "modules" / "mpi-serial2" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / "m4").exists()) - assert(not (root / "modules" / "mpi-sparse" / "README").exists()) - - diff --git a/.lib/poetry.lock b/.lib/poetry.lock deleted file mode 100644 index ac82fb0d..00000000 --- a/.lib/poetry.lock +++ /dev/null @@ -1,693 +0,0 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. - -[[package]] -name = "alabaster" -version = "0.7.13" -description = "A configurable sidebar-enabled Sphinx theme" -optional = false -python-versions = ">=3.6" -files = [ - {file = "alabaster-0.7.13-py3-none-any.whl", hash = "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3"}, - {file = "alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"}, -] - -[[package]] -name = "babel" -version = "2.15.0" -description = "Internationalization utilities" -optional = false -python-versions = ">=3.8" -files = [ - {file = "Babel-2.15.0-py3-none-any.whl", hash = "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb"}, - {file = "babel-2.15.0.tar.gz", hash = "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"}, -] - -[package.dependencies] -pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} - -[package.extras] -dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] - -[[package]] -name = "certifi" -version = "2024.8.30" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.6" -files = [ - {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, - {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.3.2" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, - {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, -] - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "docutils" -version = "0.19" -description = "Docutils -- Python Documentation Utilities" -optional = false -python-versions = ">=3.7" -files = [ - {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, - {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, -] - -[[package]] -name = "exceptiongroup" -version = "1.2.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, - {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, -] - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "fsspec" -version = "2023.12.2" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -files = [ - {file = "fsspec-2023.12.2-py3-none-any.whl", hash = "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"}, - {file = "fsspec-2023.12.2.tar.gz", hash = "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb"}, -] - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -devel = ["pytest", "pytest-cov"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -tqdm = ["tqdm"] - -[[package]] -name = "gitdb" -version = "4.0.11" -description = "Git Object Database" -optional = false -python-versions = ">=3.7" -files = [ - {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, - {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, -] - -[package.dependencies] -smmap = ">=3.0.1,<6" - -[[package]] -name = "gitpython" -version = "3.1.43" -description = "GitPython is a Python library used to interact with Git repositories" -optional = false -python-versions = ">=3.7" -files = [ - {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, - {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, -] - -[package.dependencies] -gitdb = ">=4.0.1,<5" - -[package.extras] -doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] - -[[package]] -name = "idna" -version = "3.7" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.5" -files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, -] - -[[package]] -name = "imagesize" -version = "1.4.1" -description = "Getting image size from png/jpeg/jpeg2000/gif file" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, - {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, -] - -[[package]] -name = "importlib-metadata" -version = "8.0.0" -description = "Read metadata from Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "importlib_metadata-8.0.0-py3-none-any.whl", hash = "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f"}, - {file = "importlib_metadata-8.0.0.tar.gz", hash = "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812"}, -] - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -perf = ["ipython"] -test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] - -[[package]] -name = "iniconfig" -version = "2.0.0" -description = "brain-dead simple config-ini parsing" -optional = false -python-versions = ">=3.7" -files = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, -] - -[[package]] -name = "jinja2" -version = "3.1.4" -description = "A very fast and expressive template engine." -optional = false -python-versions = ">=3.7" -files = [ - {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, - {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, -] - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "markupsafe" -version = "2.1.5" -description = "Safely add untrusted strings to HTML/XML markup." -optional = false -python-versions = ">=3.7" -files = [ - {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, - {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, -] - -[[package]] -name = "packaging" -version = "24.1" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, - {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, -] - -[[package]] -name = "pluggy" -version = "1.5.0" -description = "plugin and hook calling mechanisms for python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, - {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, -] - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "pyfakefs" -version = "5.5.0" -description = "pyfakefs implements a fake file system that mocks the Python file system modules." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pyfakefs-5.5.0-py3-none-any.whl", hash = "sha256:8dbf203ab7bef1529f11f7d41b9478b898e95bf9f3b71262163aac07a518cd76"}, - {file = "pyfakefs-5.5.0.tar.gz", hash = "sha256:7448aaa07142f892d0a4eb52a5ed3206a9f02c6599e686cd97d624c18979c154"}, -] - -[[package]] -name = "pygments" -version = "2.18.0" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.8" -files = [ - {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, - {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, -] - -[package.extras] -windows-terminal = ["colorama (>=0.4.6)"] - -[[package]] -name = "pytest" -version = "8.2.2" -description = "pytest: simple powerful testing with Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, - {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=1.5,<2.0" -tomli = {version = ">=1", markers = "python_version < \"3.11\""} - -[package.extras] -dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] - -[[package]] -name = "pytz" -version = "2024.1" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -files = [ - {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, - {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, -] - -[[package]] -name = "requests" -version = "2.32.3" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.8" -files = [ - {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, - {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, -] - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "smmap" -version = "5.0.1" -description = "A pure Python implementation of a sliding window memory map manager" -optional = false -python-versions = ">=3.7" -files = [ - {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, - {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, -] - -[[package]] -name = "snowballstemmer" -version = "2.2.0" -description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." -optional = false -python-versions = "*" -files = [ - {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, - {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, -] - -[[package]] -name = "sphinx" -version = "5.3.0" -description = "Python documentation generator" -optional = false -python-versions = ">=3.6" -files = [ - {file = "Sphinx-5.3.0.tar.gz", hash = "sha256:51026de0a9ff9fc13c05d74913ad66047e104f56a129ff73e174eb5c3ee794b5"}, - {file = "sphinx-5.3.0-py3-none-any.whl", hash = "sha256:060ca5c9f7ba57a08a1219e547b269fadf125ae25b06b9fa7f66768efb652d6d"}, -] - -[package.dependencies] -alabaster = ">=0.7,<0.8" -babel = ">=2.9" -colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} -docutils = ">=0.14,<0.20" -imagesize = ">=1.3" -importlib-metadata = {version = ">=4.8", markers = "python_version < \"3.10\""} -Jinja2 = ">=3.0" -packaging = ">=21.0" -Pygments = ">=2.12" -requests = ">=2.5.0" -snowballstemmer = ">=2.0" -sphinxcontrib-applehelp = "*" -sphinxcontrib-devhelp = "*" -sphinxcontrib-htmlhelp = ">=2.0.0" -sphinxcontrib-jsmath = "*" -sphinxcontrib-qthelp = "*" -sphinxcontrib-serializinghtml = ">=1.1.5" - -[package.extras] -docs = ["sphinxcontrib-websupport"] -lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-bugbear", "flake8-comprehensions", "flake8-simplify", "isort", "mypy (>=0.981)", "sphinx-lint", "types-requests", "types-typed-ast"] -test = ["cython", "html5lib", "pytest (>=4.6)", "typed_ast"] - -[[package]] -name = "sphinxcontrib-applehelp" -version = "1.0.4" -description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" -optional = false -python-versions = ">=3.8" -files = [ - {file = "sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e"}, - {file = "sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["pytest"] - -[[package]] -name = "sphinxcontrib-devhelp" -version = "1.0.2" -description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." -optional = false -python-versions = ">=3.5" -files = [ - {file = "sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"}, - {file = "sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["pytest"] - -[[package]] -name = "sphinxcontrib-htmlhelp" -version = "2.0.1" -description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" -optional = false -python-versions = ">=3.8" -files = [ - {file = "sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff"}, - {file = "sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["html5lib", "pytest"] - -[[package]] -name = "sphinxcontrib-jsmath" -version = "1.0.1" -description = "A sphinx extension which renders display math in HTML via JavaScript" -optional = false -python-versions = ">=3.5" -files = [ - {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, - {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, -] - -[package.extras] -test = ["flake8", "mypy", "pytest"] - -[[package]] -name = "sphinxcontrib-qthelp" -version = "1.0.3" -description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." -optional = false -python-versions = ">=3.5" -files = [ - {file = "sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72"}, - {file = "sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["pytest"] - -[[package]] -name = "sphinxcontrib-serializinghtml" -version = "1.1.5" -description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." -optional = false -python-versions = ">=3.5" -files = [ - {file = "sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952"}, - {file = "sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["pytest"] - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] - -[[package]] -name = "urllib3" -version = "2.2.2" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.8" -files = [ - {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, - {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, -] - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "wheel" -version = "0.42.0" -description = "A built-package format for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "wheel-0.42.0-py3-none-any.whl", hash = "sha256:177f9c9b0d45c47873b619f5b650346d632cdc35fb5e4d25058e09c9e581433d"}, - {file = "wheel-0.42.0.tar.gz", hash = "sha256:c45be39f7882c9d34243236f2d63cbd58039e360f85d0913425fbd7ceea617a8"}, -] - -[package.extras] -test = ["pytest (>=6.0.0)", "setuptools (>=65)"] - -[[package]] -name = "zipp" -version = "3.19.2" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.8" -files = [ - {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, - {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, -] - -[package.extras] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] - -[metadata] -lock-version = "2.0" -python-versions = "^3.8" -content-hash = "25ee2ae1d74abedde3a6637a60d4a3095ea5cf9731960875741bbc2ba84a475d" diff --git a/.lib/pyproject.toml b/.lib/pyproject.toml deleted file mode 100644 index 76ef0ef8..00000000 --- a/.lib/pyproject.toml +++ /dev/null @@ -1,42 +0,0 @@ -[tool.poetry] -name = "git-fleximod" -version = "1.0.0" -description = "Extended support for git-submodule and git-sparse-checkout" -authors = ["Jim Edwards "] -maintainers = ["Jim Edwards "] -license = "MIT" -readme = "README.md" -homepage = "https://github.com/jedwards4b/git-fleximod" -keywords = ["git", "submodule", "sparse-checkout"] -packages = [ -{ include = "git_fleximod"}, -{ include = "doc"}, -{ include = "README.md"}, -] - -[tool.poetry.scripts] -git-fleximod = "git_fleximod.git_fleximod:main" -me2flexi = "git_fleximod.metoflexi:_main" -fsspec = "fsspec.fuse:main" - -[tool.poetry.dependencies] -python = "^3.8" -GitPython = "^3.1.0" -sphinx = "^5.0.0" -fsspec = "^2023.12.2" -wheel = "^0.42.0" -pytest = "^8.0.0" -pyfakefs = "^5.3.5" - -[tool.poetry.urls] -"Bug Tracker" = "https://github.com/jedwards4b/git-fleximod/issues" - -[tool.pytest.ini_options] -markers = [ - "skip_after_first: only run on first iteration" -] - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" - diff --git a/.lib/tbump.toml b/.lib/tbump.toml deleted file mode 100644 index 4ac513dd..00000000 --- a/.lib/tbump.toml +++ /dev/null @@ -1,43 +0,0 @@ -# Uncomment this if your project is hosted on GitHub: -github_url = "https://github.com/jedwards4b/git-fleximod/" - -[version] -current = "1.0.0" - -# Example of a semver regexp. -# Make sure this matches current_version before -# using tbump -regex = ''' - (?P\d+) - \. - (?P\d+) - \. - (?P\d+) - ''' - -[git] -message_template = "Bump to {new_version}" -tag_template = "v{new_version}" - -# For each file to patch, add a [[file]] config -# section containing the path of the file, relative to the -# tbump.toml location. -[[file]] -src = "git_fleximod/cli.py" - -[[file]] -src = "pyproject.toml" - -# You can specify a list of commands to -# run after the files have been patched -# and before the git commit is made - -# [[before_commit]] -# name = "check changelog" -# cmd = "grep -q {new_version} Changelog.rst" - -# Or run some commands after the git tag and the branch -# have been pushed: -# [[after_push]] -# name = "publish" -# cmd = "./publish.sh" diff --git a/.lib/tests/__init__.py b/.lib/tests/__init__.py deleted file mode 100644 index 4d4c66c7..00000000 --- a/.lib/tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -import sys, os - -sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir, "src")) diff --git a/.lib/tests/conftest.py b/.lib/tests/conftest.py deleted file mode 100644 index 44d28e17..00000000 --- a/.lib/tests/conftest.py +++ /dev/null @@ -1,150 +0,0 @@ -import pytest -from git_fleximod.gitinterface import GitInterface -import os -import subprocess -import logging -from pathlib import Path - -@pytest.fixture(scope='session') -def logger(): - logging.basicConfig( - level=logging.INFO, format="%(name)s - %(levelname)s - %(message)s", handlers=[logging.StreamHandler()] - ) - logger = logging.getLogger(__name__) - return logger - -all_repos=[ - {"subrepo_path": "modules/test", - "submodule_name": "test_submodule", - "status1" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", - "status2" : "test_submodule at tag MPIserial_2.4.0", - "status3" : "test_submodule at tag MPIserial_2.4.0", - "status4" : "test_submodule at tag MPIserial_2.4.0", - "gitmodules_content" : """ - [submodule "test_submodule"] - path = modules/test - url = https://github.com/ESMCI/mpi-serial.git - fxtag = MPIserial_2.4.0 - fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = ToplevelRequired -"""}, - {"subrepo_path": "modules/test_optional", - "submodule_name": "test_optional", - "status1" : "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", - "status2" : "test_optional at tag MPIserial_2.4.0", - "status3" : "test_optional not checked out, out of sync at tag MPIserial_2.5.4, expected tag is MPIserial_2.4.0 (optional)", - "status4" : "test_optional at tag MPIserial_2.4.0", - "gitmodules_content": """ - [submodule "test_optional"] - path = modules/test_optional - url = https://github.com/ESMCI/mpi-serial.git - fxtag = MPIserial_2.4.0 - fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = ToplevelOptional -"""}, - {"subrepo_path": "modules/test_alwaysoptional", - "submodule_name": "test_alwaysoptional", - "status1" : "test_alwaysoptional MPIserial_2.3.0 is out of sync with .gitmodules e5cf35c", - "status2" : "test_alwaysoptional at hash e5cf35c", - "status3" : "out of sync at tag MPIserial_2.5.4, expected tag is e5cf35c", - "status4" : "test_alwaysoptional at hash e5cf35c", - "gitmodules_content": """ - [submodule "test_alwaysoptional"] - path = modules/test_alwaysoptional - url = https://github.com/ESMCI/mpi-serial.git - fxtag = e5cf35c - fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = AlwaysOptional -"""}, - {"subrepo_path": "modules/test_sparse", - "submodule_name": "test_sparse", - "status1" : "test_sparse at tag MPIserial_2.5.0", - "status2" : "test_sparse at tag MPIserial_2.5.0", - "status3" : "test_sparse at tag MPIserial_2.5.0", - "status4" : "test_sparse at tag MPIserial_2.5.0", - "gitmodules_content": """ - [submodule "test_sparse"] - path = modules/test_sparse - url = https://github.com/ESMCI/mpi-serial.git - fxtag = MPIserial_2.5.0 - fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = AlwaysRequired - fxsparse = ../.sparse_file_list -"""}, -] -@pytest.fixture(params=all_repos) - -def shared_repos(request): - return request.param - -@pytest.fixture -def get_all_repos(): - return all_repos - -def write_sparse_checkout_file(fp): - sparse_content = """m4 -""" - fp.write_text(sparse_content) - -@pytest.fixture -def test_repo(shared_repos, tmp_path, logger): - subrepo_path = shared_repos["subrepo_path"] - submodule_name = shared_repos["submodule_name"] - test_dir = tmp_path / "testrepo" - test_dir.mkdir() - str_path = str(test_dir) - gitp = GitInterface(str_path, logger) - assert test_dir.joinpath(".git").is_dir() - (test_dir / "modules").mkdir() - if "sparse" in submodule_name: - (test_dir / subrepo_path).mkdir() - # Add the sparse checkout file - write_sparse_checkout_file(test_dir / "modules" / ".sparse_file_list") - gitp.git_operation("add","modules/.sparse_file_list") - else: - gitp = GitInterface(str(test_dir), logger) - gitp.git_operation("submodule", "add", "--depth","1","--name", submodule_name, "https://github.com/ESMCI/mpi-serial.git", subrepo_path) - assert test_dir.joinpath(".gitmodules").is_file() - gitp.git_operation("add",subrepo_path) - gitp.git_operation("commit","-a","-m","\"add submod\"") - test_dir2 = tmp_path / "testrepo2" - gitp.git_operation("clone",test_dir,test_dir2) - return test_dir2 - - -@pytest.fixture -def complex_repo(tmp_path, logger): - test_dir = tmp_path / "testcomplex" - test_dir.mkdir() - str_path = str(test_dir) - gitp = GitInterface(str_path, logger) - gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") - gitp.git_operation("fetch", "origin") - gitp.git_operation("checkout", "v0.0.1") - return test_dir - -@pytest.fixture -def complex_update(tmp_path, logger): - test_dir = tmp_path / "testcomplex" - test_dir.mkdir() - str_path = str(test_dir) - gitp = GitInterface(str_path, logger) - gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") - gitp.git_operation("fetch", "origin") - gitp.git_operation("checkout", "v0.0.2") - - return test_dir - -@pytest.fixture -def git_fleximod(): - def _run_fleximod(path, args, input=None): - cmd = ["git", "fleximod"] + args.split() - result = subprocess.run(cmd, cwd=path, input=input, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True) - if result.returncode: - print(result.stdout) - print(result.stderr) - return result - return _run_fleximod - diff --git a/.lib/tests/test_a_import.py b/.lib/tests/test_a_import.py deleted file mode 100644 index d5ca878d..00000000 --- a/.lib/tests/test_a_import.py +++ /dev/null @@ -1,8 +0,0 @@ -# pylint: disable=unused-import -from git_fleximod import cli -from git_fleximod import utils -from git_fleximod.gitinterface import GitInterface -from git_fleximod.gitmodules import GitModules - -def test_import(): - print("here") diff --git a/.lib/tests/test_b_update.py b/.lib/tests/test_b_update.py deleted file mode 100644 index 159f1cfa..00000000 --- a/.lib/tests/test_b_update.py +++ /dev/null @@ -1,26 +0,0 @@ -import pytest -from pathlib import Path - -def test_basic_checkout(git_fleximod, test_repo, shared_repos): - # Prepare a simple .gitmodules - gm = shared_repos['gitmodules_content'] - file_path = (test_repo / ".gitmodules") - repo_name = shared_repos["submodule_name"] - repo_path = shared_repos["subrepo_path"] - - file_path.write_text(gm) - - # Run the command - result = git_fleximod(test_repo, f"update {repo_name}") - - # Assertions - assert result.returncode == 0 - assert Path(test_repo / repo_path).exists() # Did the submodule directory get created? - if "sparse" in repo_name: - assert Path(test_repo / f"{repo_path}/m4").exists() # Did the submodule sparse directory get created? - assert not Path(test_repo / f"{repo_path}/README").exists() # Did only the submodule sparse directory get created? - - status = git_fleximod(test_repo, f"status {repo_name}") - - assert shared_repos["status2"] in status.stdout - diff --git a/.lib/tests/test_c_required.py b/.lib/tests/test_c_required.py deleted file mode 100644 index 89ab8d29..00000000 --- a/.lib/tests/test_c_required.py +++ /dev/null @@ -1,30 +0,0 @@ -import pytest -from pathlib import Path - -def test_required(git_fleximod, test_repo, shared_repos): - file_path = (test_repo / ".gitmodules") - gm = shared_repos["gitmodules_content"] - repo_name = shared_repos["submodule_name"] - if file_path.exists(): - with file_path.open("r") as f: - gitmodules_content = f.read() - # add the entry if it does not exist - if repo_name not in gitmodules_content: - file_path.write_text(gitmodules_content+gm) - # or if it is incomplete - elif gm not in gitmodules_content: - file_path.write_text(gm) - else: - file_path.write_text(gm) - result = git_fleximod(test_repo, "update") - assert result.returncode == 0 - status = git_fleximod(test_repo, f"status {repo_name}") - assert shared_repos["status3"] in status.stdout - status = git_fleximod(test_repo, f"update --optional") - assert result.returncode == 0 - status = git_fleximod(test_repo, f"status {repo_name}") - assert shared_repos["status4"] in status.stdout - status = git_fleximod(test_repo, f"update {repo_name}") - assert result.returncode == 0 - status = git_fleximod(test_repo, f"status {repo_name}") - assert shared_repos["status4"] in status.stdout diff --git a/.lib/tests/test_d_complex.py b/.lib/tests/test_d_complex.py deleted file mode 100644 index edde7d81..00000000 --- a/.lib/tests/test_d_complex.py +++ /dev/null @@ -1,66 +0,0 @@ -import pytest -from pathlib import Path -from git_fleximod.gitinterface import GitInterface - -def test_complex_checkout(git_fleximod, complex_repo, logger): - status = git_fleximod(complex_repo, "status") - assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) - assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout) - assert("Complex not checked out, aligned at tag testtag02" in status.stdout) - assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) - - # This should checkout and update test_submodule and complex_sub - result = git_fleximod(complex_repo, "update") - assert result.returncode == 0 - - status = git_fleximod(complex_repo, "status") - assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag02" in status.stdout) - - # now check the complex_sub - root = (complex_repo / "modules" / "complex") - assert(not (root / "libraries" / "gptl" / ".git").exists()) - assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serial" / ".git").exists()) - assert(not (root / "modules" / "mpi-serial2" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / "m4").exists()) - assert(not (root / "modules" / "mpi-sparse" / "README").exists()) - - # update a single optional submodule - - result = git_fleximod(complex_repo, "update ToplevelOptional") - assert result.returncode == 0 - - status = git_fleximod(complex_repo, "status") - assert("ToplevelOptional at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag02" in status.stdout) - assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) - - # Finally update optional - result = git_fleximod(complex_repo, "update --optional") - assert result.returncode == 0 - - status = git_fleximod(complex_repo, "status") - assert("ToplevelOptional at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag02" in status.stdout) - assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout) - - # now check the complex_sub - root = (complex_repo / "modules" / "complex" ) - assert(not (root / "libraries" / "gptl" / ".git").exists()) - assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serial2" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / "m4").exists()) - assert(not (root / "modules" / "mpi-sparse" / "README").exists()) - - diff --git a/.lib/tests/test_e_complex_update.py b/.lib/tests/test_e_complex_update.py deleted file mode 100644 index 0c3ab4c6..00000000 --- a/.lib/tests/test_e_complex_update.py +++ /dev/null @@ -1,69 +0,0 @@ -import pytest -from pathlib import Path -from git_fleximod.gitinterface import GitInterface - -def test_complex_update(git_fleximod, complex_update, logger): - status = git_fleximod(complex_update, "status") - assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) - assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout) - assert("Complex not checked out, out of sync at tag testtag02, expected tag is testtag3" in status.stdout) - assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) - - # This should checkout and update test_submodule and complex_sub - result = git_fleximod(complex_update, "update") - assert result.returncode == 0 - - status = git_fleximod(complex_update, "status") - assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag3" in status.stdout) - - # now check the complex_sub - root = (complex_update / "modules" / "complex") - assert(not (root / "libraries" / "gptl" / ".git").exists()) - assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serialAR" / ".git").exists()) - assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) - assert(not (root / "modules" / "mpi-serial2" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / "m4").exists()) - assert(not (root / "modules" / "mpi-sparse" / "README").exists()) - - # update a single optional submodule - - result = git_fleximod(complex_update, "update ToplevelOptional") - assert result.returncode == 0 - - status = git_fleximod(complex_update, "status") - assert("ToplevelOptional at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag3" in status.stdout) - assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) - - # Finally update optional - result = git_fleximod(complex_update, "update --optional") - assert result.returncode == 0 - - status = git_fleximod(complex_update, "status") - assert("ToplevelOptional at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag3" in status.stdout) - assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout) - - # now check the complex_sub - root = (complex_update / "modules" / "complex" ) - assert(not (root / "libraries" / "gptl" / ".git").exists()) - assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) - assert(not (root / "modules" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serialAR" / ".git").exists()) - assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / ".git").exists()) - assert((root / "modules" / "mpi-serial2" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / "m4").exists()) - assert(not (root / "modules" / "mpi-sparse" / "README").exists()) - - From 3dd5cadd2c405b84426df3e738c53ff3f182d281 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Mon, 13 Oct 2025 20:39:07 -0600 Subject: [PATCH 097/150] remove correct git-fleximod this time --- .lib/.github/workflows/pre-commit | 13 + .lib/.github/workflows/pytest.yaml | 77 ++ .lib/.pre-commit-config.yaml | 18 + .../git-fleximod/.github/workflows/pre-commit | 13 + .../.github/workflows/pytest.yaml | 77 ++ .lib/git-fleximod/.pre-commit-config.yaml | 18 + .lib/git-fleximod/CODE_OF_CONDUCT.md | 107 +++ .lib/git-fleximod/License | 20 + .lib/git-fleximod/README.md | 108 +++ .lib/git-fleximod/doc/Makefile | 20 + .lib/git-fleximod/doc/conf.py | 26 + .lib/git-fleximod/doc/index.rst | 24 + .lib/git-fleximod/doc/make.bat | 35 + .lib/git-fleximod/escomp_install | 25 + .../git_fleximod/__init__.py | 0 .lib/{ => git-fleximod}/git_fleximod/cli.py | 42 +- .../git_fleximod/git_fleximod.py | 0 .../git_fleximod/gitinterface.py | 9 +- .../git_fleximod/gitmodules.py | 0 .../git_fleximod/lstripreader.py | 0 .../git_fleximod/metoflexi.py | 0 .../git_fleximod/submodule.py | 99 ++- .lib/{ => git-fleximod}/git_fleximod/utils.py | 0 .lib/git-fleximod/poetry.lock | 693 ++++++++++++++++++ .lib/git-fleximod/pyproject.toml | 42 ++ .lib/git-fleximod/tbump.toml | 43 ++ .lib/git-fleximod/tests/__init__.py | 3 + .lib/git-fleximod/tests/conftest.py | 150 ++++ .lib/git-fleximod/tests/test_a_import.py | 8 + .lib/git-fleximod/tests/test_b_update.py | 26 + .lib/git-fleximod/tests/test_c_required.py | 43 ++ .lib/git-fleximod/tests/test_d_complex.py | 66 ++ .../tests/test_e_complex_update.py | 69 ++ 33 files changed, 1815 insertions(+), 59 deletions(-) create mode 100644 .lib/.github/workflows/pre-commit create mode 100644 .lib/.github/workflows/pytest.yaml create mode 100644 .lib/.pre-commit-config.yaml create mode 100644 .lib/git-fleximod/.github/workflows/pre-commit create mode 100644 .lib/git-fleximod/.github/workflows/pytest.yaml create mode 100644 .lib/git-fleximod/.pre-commit-config.yaml create mode 100644 .lib/git-fleximod/CODE_OF_CONDUCT.md create mode 100644 .lib/git-fleximod/License create mode 100644 .lib/git-fleximod/README.md create mode 100644 .lib/git-fleximod/doc/Makefile create mode 100644 .lib/git-fleximod/doc/conf.py create mode 100644 .lib/git-fleximod/doc/index.rst create mode 100644 .lib/git-fleximod/doc/make.bat create mode 100644 .lib/git-fleximod/escomp_install rename .lib/{ => git-fleximod}/git_fleximod/__init__.py (100%) rename .lib/{ => git-fleximod}/git_fleximod/cli.py (81%) rename .lib/{ => git-fleximod}/git_fleximod/git_fleximod.py (100%) rename .lib/{ => git-fleximod}/git_fleximod/gitinterface.py (93%) rename .lib/{ => git-fleximod}/git_fleximod/gitmodules.py (100%) rename .lib/{ => git-fleximod}/git_fleximod/lstripreader.py (100%) rename .lib/{ => git-fleximod}/git_fleximod/metoflexi.py (100%) rename .lib/{ => git-fleximod}/git_fleximod/submodule.py (88%) rename .lib/{ => git-fleximod}/git_fleximod/utils.py (100%) create mode 100644 .lib/git-fleximod/poetry.lock create mode 100644 .lib/git-fleximod/pyproject.toml create mode 100644 .lib/git-fleximod/tbump.toml create mode 100644 .lib/git-fleximod/tests/__init__.py create mode 100644 .lib/git-fleximod/tests/conftest.py create mode 100644 .lib/git-fleximod/tests/test_a_import.py create mode 100644 .lib/git-fleximod/tests/test_b_update.py create mode 100644 .lib/git-fleximod/tests/test_c_required.py create mode 100644 .lib/git-fleximod/tests/test_d_complex.py create mode 100644 .lib/git-fleximod/tests/test_e_complex_update.py diff --git a/.lib/.github/workflows/pre-commit b/.lib/.github/workflows/pre-commit new file mode 100644 index 00000000..1a6ad008 --- /dev/null +++ b/.lib/.github/workflows/pre-commit @@ -0,0 +1,13 @@ +name: pre-commit +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.0 diff --git a/.lib/.github/workflows/pytest.yaml b/.lib/.github/workflows/pytest.yaml new file mode 100644 index 00000000..0868dd9a --- /dev/null +++ b/.lib/.github/workflows/pytest.yaml @@ -0,0 +1,77 @@ +# Run this job on pushes to `main`, and for pull requests. If you don't specify +# `branches: [main], then this actions runs _twice_ on pull requests, which is +# annoying. + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # If you wanted to use multiple Python versions, you'd have specify a matrix in the job and + # reference the matrixe python version here. + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + + # Cache the installation of Poetry itself, e.g. the next step. This prevents the workflow + # from installing Poetry every time, which can be slow. Note the use of the Poetry version + # number in the cache key, and the "-0" suffix: this allows you to invalidate the cache + # manually if/when you want to upgrade Poetry, or if something goes wrong. This could be + # mildly cleaner by using an environment variable, but I don't really care. + - name: cache poetry install + uses: actions/cache@v4 + with: + path: ~/.local + key: poetry-1.7.1 + + # Install Poetry. You could do this manually, or there are several actions that do this. + # `snok/install-poetry` seems to be minimal yet complete, and really just calls out to + # Poetry's default install script, which feels correct. I pin the Poetry version here + # because Poetry does occasionally change APIs between versions and I don't want my + # actions to break if it does. + # + # The key configuration value here is `virtualenvs-in-project: true`: this creates the + # venv as a `.venv` in your testing directory, which allows the next step to easily + # cache it. + - uses: snok/install-poetry@v1 + with: + version: 1.7.1 + virtualenvs-create: true + virtualenvs-in-project: true + + # Cache your dependencies (i.e. all the stuff in your `pyproject.toml`). Note the cache + # key: if you're using multiple Python versions, or multiple OSes, you'd need to include + # them in the cache key. I'm not, so it can be simple and just depend on the poetry.lock. + - name: cache deps + id: cache-deps + uses: actions/cache@v4 + with: + path: .venv + key: pydeps-${{ hashFiles('**/poetry.lock') }} + + # Install dependencies. `--no-root` means "install all dependencies but not the project + # itself", which is what you want to avoid caching _your_ code. The `if` statement + # ensures this only runs on a cache miss. + - run: poetry install --no-interaction --no-root + if: steps.cache-deps.outputs.cache-hit != 'true' + + # Now install _your_ project. This isn't necessary for many types of projects -- particularly + # things like Django apps don't need this. But it's a good idea since it fully-exercises the + # pyproject.toml and makes that if you add things like console-scripts at some point that + # they'll be installed and working. + - run: poetry install --no-interaction + + # And finally run tests. I'm using pytest and all my pytest config is in my `pyproject.toml` + # so this line is super-simple. But it could be as complex as you need. + - run: | + git config --global user.name "${GITHUB_ACTOR}" + git config --global user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" + poetry run pytest + diff --git a/.lib/.pre-commit-config.yaml b/.lib/.pre-commit-config.yaml new file mode 100644 index 00000000..2f6089da --- /dev/null +++ b/.lib/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +exclude: ^utils/.*$ + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + - repo: https://github.com/PyCQA/pylint + rev: v2.11.1 + hooks: + - id: pylint + args: + - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error,no-member diff --git a/.lib/git-fleximod/.github/workflows/pre-commit b/.lib/git-fleximod/.github/workflows/pre-commit new file mode 100644 index 00000000..1a6ad008 --- /dev/null +++ b/.lib/git-fleximod/.github/workflows/pre-commit @@ -0,0 +1,13 @@ +name: pre-commit +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.0 diff --git a/.lib/git-fleximod/.github/workflows/pytest.yaml b/.lib/git-fleximod/.github/workflows/pytest.yaml new file mode 100644 index 00000000..0868dd9a --- /dev/null +++ b/.lib/git-fleximod/.github/workflows/pytest.yaml @@ -0,0 +1,77 @@ +# Run this job on pushes to `main`, and for pull requests. If you don't specify +# `branches: [main], then this actions runs _twice_ on pull requests, which is +# annoying. + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # If you wanted to use multiple Python versions, you'd have specify a matrix in the job and + # reference the matrixe python version here. + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + + # Cache the installation of Poetry itself, e.g. the next step. This prevents the workflow + # from installing Poetry every time, which can be slow. Note the use of the Poetry version + # number in the cache key, and the "-0" suffix: this allows you to invalidate the cache + # manually if/when you want to upgrade Poetry, or if something goes wrong. This could be + # mildly cleaner by using an environment variable, but I don't really care. + - name: cache poetry install + uses: actions/cache@v4 + with: + path: ~/.local + key: poetry-1.7.1 + + # Install Poetry. You could do this manually, or there are several actions that do this. + # `snok/install-poetry` seems to be minimal yet complete, and really just calls out to + # Poetry's default install script, which feels correct. I pin the Poetry version here + # because Poetry does occasionally change APIs between versions and I don't want my + # actions to break if it does. + # + # The key configuration value here is `virtualenvs-in-project: true`: this creates the + # venv as a `.venv` in your testing directory, which allows the next step to easily + # cache it. + - uses: snok/install-poetry@v1 + with: + version: 1.7.1 + virtualenvs-create: true + virtualenvs-in-project: true + + # Cache your dependencies (i.e. all the stuff in your `pyproject.toml`). Note the cache + # key: if you're using multiple Python versions, or multiple OSes, you'd need to include + # them in the cache key. I'm not, so it can be simple and just depend on the poetry.lock. + - name: cache deps + id: cache-deps + uses: actions/cache@v4 + with: + path: .venv + key: pydeps-${{ hashFiles('**/poetry.lock') }} + + # Install dependencies. `--no-root` means "install all dependencies but not the project + # itself", which is what you want to avoid caching _your_ code. The `if` statement + # ensures this only runs on a cache miss. + - run: poetry install --no-interaction --no-root + if: steps.cache-deps.outputs.cache-hit != 'true' + + # Now install _your_ project. This isn't necessary for many types of projects -- particularly + # things like Django apps don't need this. But it's a good idea since it fully-exercises the + # pyproject.toml and makes that if you add things like console-scripts at some point that + # they'll be installed and working. + - run: poetry install --no-interaction + + # And finally run tests. I'm using pytest and all my pytest config is in my `pyproject.toml` + # so this line is super-simple. But it could be as complex as you need. + - run: | + git config --global user.name "${GITHUB_ACTOR}" + git config --global user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" + poetry run pytest + diff --git a/.lib/git-fleximod/.pre-commit-config.yaml b/.lib/git-fleximod/.pre-commit-config.yaml new file mode 100644 index 00000000..2f6089da --- /dev/null +++ b/.lib/git-fleximod/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +exclude: ^utils/.*$ + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + - repo: https://github.com/PyCQA/pylint + rev: v2.11.1 + hooks: + - id: pylint + args: + - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error,no-member diff --git a/.lib/git-fleximod/CODE_OF_CONDUCT.md b/.lib/git-fleximod/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..84f2925b --- /dev/null +++ b/.lib/git-fleximod/CODE_OF_CONDUCT.md @@ -0,0 +1,107 @@ +# Contributor Code of Conduct +_The Contributor Code of Conduct is for participants in our software projects and community._ + +## Our Pledge +We, as contributors, creators, stewards, and maintainers (participants), of **git-fleximod** pledge to make participation in +our software, system or hardware project and community a safe, productive, welcoming and inclusive experience for everyone. +All participants are required to abide by this Code of Conduct. +This includes respectful treatment of everyone regardless of age, body size, disability, ethnicity, gender identity or expression, +level of experience, nationality, political affiliation, veteran status, pregnancy, genetic information, physical appearance, race, +religion, or sexual orientation, as well as any other characteristic protected under applicable US federal or state law. + +## Our Standards +Examples of behaviors that contribute to a positive environment include: + +* All participants are treated with respect and consideration, valuing a diversity of views and opinions +* Be considerate, respectful, and collaborative +* Communicate openly with respect for others, critiquing ideas rather than individuals and gracefully accepting criticism +* Acknowledging the contributions of others +* Avoid personal attacks directed toward other participants +* Be mindful of your surroundings and of your fellow participants +* Alert UCAR staff and suppliers/vendors if you notice a dangerous situation or someone in distress +* Respect the rules and policies of the project and venue + +Examples of unacceptable behavior include, but are not limited to: + +* Harassment, intimidation, or discrimination in any form +* Physical, verbal, or written abuse by anyone to anyone, including repeated use of pronouns other than those requested +* Unwelcome sexual attention or advances +* Personal attacks directed at other guests, members, participants, etc. +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Alarming, intimidating, threatening, or hostile comments or conduct +* Inappropriate use of nudity and/or sexual images +* Threatening or stalking anyone, including a participant +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Scope +This Code of Conduct applies to all spaces managed by the Project whether they be physical, online or face-to-face. +This includes project code, code repository, associated web pages, documentation, mailing lists, project websites and wiki pages, +issue tracker, meetings, telecons, events, project social media accounts, and any other forums created by the project team which the +community uses for communication. +In addition, violations of this Code of Conduct outside these spaces may affect a person's ability to participate within them. +Representation of a project may be further defined and clarified by project maintainers. + +## Community Responsibilities +Everyone in the community is empowered to respond to people who are showing unacceptable behavior. +They can talk to them privately or publicly. +Anyone requested to stop unacceptable behavior is expected to comply immediately. +If the behavior continues concerns may be brought to the project administrators or to any other party listed in the +[Reporting](#reporting) section below. + +## Project Administrator Responsibilities +Project administrators are responsible for clarifying the standards of acceptable behavior and are encouraged to model appropriate +behavior and provide support when people in the community point out inappropriate behavior. +Project administrator(s) are normally the ones that would be tasked to carry out the actions in the [Consequences](#consequences) +section below. + +Project administrators are also expected to keep this Code of Conduct updated with the main one housed at UCAR, as listed below in +the [Attribution](#attribution) section. + +## Reporting +Instances of unacceptable behavior can be brought to the attention of the project administrator(s) who may take any action as +outlined in the [Consequences](#consequences) section below. +However, making a report to a project administrator is not considered an 'official report' to UCAR. + +Instances of unacceptable behavior may also be reported directly to UCAR pursuant to [UCAR's Harassment Reporting and Complaint +Procedure](https://www2.fin.ucar.edu/procedures/hr/harassment-reporting-and-complaint-procedure), or anonymously through [UCAR's +EthicsPoint Hotline](https://www2.fin.ucar.edu/ethics/anonymous-reporting). + +Complaints received by UCAR will be handled pursuant to the procedures outlined in UCAR's Harassment Reporting and Complaint +Procedure. +Complaints to UCAR will be held as confidential as practicable under the circumstances, and retaliation against a person who +initiates a complaint or an inquiry about inappropriate behavior will not be tolerated. + +Any Contributor can use these reporting methods even if they are not directly affiliated with UCAR. +The Frequently Asked Questions (FAQ) page for reporting is [here](https://www2.fin.ucar.edu/procedures/hr/reporting-faqs). + +## Consequences +Upon receipt of a complaint, the project administrator(s) may take any action deemed necessary and appropriate under the +circumstances. +Such action can include things such as: removing, editing, or rejecting comments, commits, code, wiki edits, email, issues, and +other contributions that are not aligned to this Code of Conduct, or banning temporarily or permanently any contributor for other +behaviors that are deemed inappropriate, threatening, offensive, or harmful. +Project administrators also have the right to report violations to UCAR HR and/or UCAR's Office of Diversity, Equity and Inclusion +(ODEI), as well as a participant's home institution and/or law enforcement. +In the event an incident is reported to UCAR, UCAR will follow its Harassment Reporting and Complaint Procedure. + +## Process for Changes +All UCAR managed projects are required to adopt this Contributor Code of Conduct. +Adoption is assumed even if not expressly stated in the repository. +Projects should fill in sections where prompted with project-specific information, including, project name and adoption date. + +Projects that adopt this Code of Conduct need to stay up to date with UCAR's Contributor Code of Conduct, linked with a DOI in the +[Attribution](#attribution) section below. +Projects can make limited substantive changes to the Code of Conduct, however, the changes must be limited in scope and may not +contradict the UCAR Contributor Code of Conduct. + +## Attribution +This Code of Conduct was originally adapted from the [Contributor Covenant](http://contributor-covenant.org/version/1/4), version +1.4. +We then aligned it with the UCAR Participant Code of Conduct, which also borrows from the American Geophysical Union (AGU) Code of +Conduct. +The UCAR Participant Code of Conduct applies to both UCAR employees as well as participants in activities run by UCAR. +The original version of this for all software projects that have strong management from UCAR or UCAR staff is available on the UCAR +website at https://doi.org/10.5065/6w2c-a132. +The date that it was adopted by this project was **Feb/13/2018**. +When responding to complaints, UCAR HR and ODEI will do so based on the latest published version. +Therefore, any project-specific changes should follow the [Process for Changes](#process-for-changes) section above. diff --git a/.lib/git-fleximod/License b/.lib/git-fleximod/License new file mode 100644 index 00000000..88bc2251 --- /dev/null +++ b/.lib/git-fleximod/License @@ -0,0 +1,20 @@ +Copyright 2024 NSF National Center for Atmospheric Sciences (NCAR) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +“Software”), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/.lib/git-fleximod/README.md b/.lib/git-fleximod/README.md new file mode 100644 index 00000000..53917da4 --- /dev/null +++ b/.lib/git-fleximod/README.md @@ -0,0 +1,108 @@ +# git-fleximod + +Flexible, Enhanced Submodule Management for Git + +## Overview + +Git-fleximod is a Python-based tool that extends Git's submodule and sparse checkout capabilities, offering additional features for managing submodules in a more flexible and efficient way. + +## Installation + + If you choose to locate git-fleximod in your path you can access it via command: git fleximod + +## Usage + + Basic Usage: + git fleximod [options] + Available Commands: + status: Display the status of submodules. + update: Update submodules to the tag indicated in .gitmodules variable fxtag. + test: Make sure that fxtags and submodule hashes are consistant, + make sure that official urls (as defined by fxDONOTUSEurl) are set + make sure that fxtags are defined for all submodules + Additional Options: + See git fleximod --help for more details. + +## Supported .gitmodules Variables + + fxtag: Specify a specific tag or branch to checkout for a submodule. + fxrequired: Mark a submodule's checkout behavior, with allowed values: + - ToplevelRequired: Top-level and required (checked out only when this is the Toplevel module). + - ToplevelOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module). + - AlwaysRequired: Always required (always checked out). + - AlwaysOptional: Always optional (checked out with --optional flag). + fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths. + fxDONOTUSEurl: This is the url used in the test subcommand to assure that protected branches do not point to forks + **NOTE** the fxDONOTUSEurl variable is only used to identify the official project repository and should not be + changed by users. Use the url variable to change to a fork if desired. + +## Sparse Checkouts + + To enable sparse checkout for a submodule, set the fxsparse variable + in the .gitmodules file to the path of a file containing the desired + sparse checkout paths. Git-fleximod will automatically configure + sparse checkout based on this file when applicable commands are run. + See [git-sparse-checkout](https://git-scm.com/docs/git-sparse-checkout#_internalsfull_pattern_set) + for details on the format of this file. + +## Tests + + The git fleximod test action is designed to be used by, for example, github workflows + to assure that protected branches are consistant with respect to submodule hashes and fleximod fxtags + +## Examples + +Here are some common usage examples: + +Update all submodules, including optional ones: +```bash + git fleximod update --optional +``` + +Updating a specific submodule to the fxtag indicated in .gitmodules: + +```bash + git fleximod update submodule-name +``` +Example .gitmodules entry: +```ini, toml + [submodule "cosp2"] + path = src/physics/cosp2/src + url = https://github.com/CFMIP/COSPv2.0 + fxsparse = ../.cosp_sparse_checkout + fxrequired = AlwaysRequired + fxtag = v2.1.4cesm +``` +Explanation: + +This entry indicates that the submodule named cosp2 at tag v2.1.4cesm +should be checked out into the directory src/physics/cosp2/src +relative to the .gitmodules directory. It should be checked out from +the URL https://github.com/CFMIP/COSPv2.0 and use sparse checkout as +described in the file ../.cosp_sparse_checkout relative to the path +directory. It should be checked out anytime this .gitmodules entry is +read. + +Additional example: +```ini, toml + [submodule "cime"] + path = cime + url = https://github.com/jedwards4b/cime + fxrequired = ToplevelRequired + fxtag = cime6.0.198_rme01 +``` + +Explanation: + +This entry indicates that the submodule cime should be checked out +into a directory named cime at tag cime6.0.198_rme01 from the URL +https://github.com/jedwards4b/cime. This should only be done if +the .gitmodules file is at the top level of the repository clone. + +## Contributing + +We welcome contributions! Please see the CONTRIBUTING.md file for guidelines. + +## License + +Git-fleximod is released under the MIT License. diff --git a/.lib/git-fleximod/doc/Makefile b/.lib/git-fleximod/doc/Makefile new file mode 100644 index 00000000..d4bb2cbb --- /dev/null +++ b/.lib/git-fleximod/doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/.lib/git-fleximod/doc/conf.py b/.lib/git-fleximod/doc/conf.py new file mode 100644 index 00000000..423099ee --- /dev/null +++ b/.lib/git-fleximod/doc/conf.py @@ -0,0 +1,26 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "git-fleximod" +author = "Jim Edwards " +release = "0.4.0" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ["sphinx_argparse_cli"] + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "alabaster" +html_static_path = ["_static"] diff --git a/.lib/git-fleximod/doc/index.rst b/.lib/git-fleximod/doc/index.rst new file mode 100644 index 00000000..0f9c1a7f --- /dev/null +++ b/.lib/git-fleximod/doc/index.rst @@ -0,0 +1,24 @@ +.. git-fleximod documentation master file, created by + sphinx-quickstart on Sat Feb 3 12:02:22 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to git-fleximod's documentation! +======================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: +.. module:: sphinxcontrib.autoprogram +.. sphinx_argparse_cli:: + :module: git_fleximod.cli + :func: get_parser + :prog: git-fleximod + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/.lib/git-fleximod/doc/make.bat b/.lib/git-fleximod/doc/make.bat new file mode 100644 index 00000000..32bb2452 --- /dev/null +++ b/.lib/git-fleximod/doc/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/.lib/git-fleximod/escomp_install b/.lib/git-fleximod/escomp_install new file mode 100644 index 00000000..ae782e72 --- /dev/null +++ b/.lib/git-fleximod/escomp_install @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# updates git-fleximod in an ESCOMP model +# this script should be run from the model root directory, it expects +# git-fleximod to already be installed with the script in bin +# and the classes in lib/python/site-packages +import sys +import shutil +import os + +from glob import iglob + +fleximod_root = sys.argv[1] +fleximod_path = os.path.join(fleximod_root,"src","git-fleximod") +if os.path.isfile(fleximod_path): + with open(fleximod_path,"r") as f: + fleximod = f.readlines() + with open(os.path.join(".","bin","git-fleximod"),"w") as f: + for line in fleximod: + f.write(line) + if "import argparse" in line: + f.write('\nsys.path.append(os.path.join(os.path.dirname(__file__),"..","lib","python","site-packages"))\n\n') + + for file in iglob(os.path.join(fleximod_root, "src", "fleximod", "*.py")): + shutil.copy(file, + os.path.join("lib","python","site-packages","fleximod",os.path.basename(file))) diff --git a/.lib/git_fleximod/__init__.py b/.lib/git-fleximod/git_fleximod/__init__.py similarity index 100% rename from .lib/git_fleximod/__init__.py rename to .lib/git-fleximod/git_fleximod/__init__.py diff --git a/.lib/git_fleximod/cli.py b/.lib/git-fleximod/git_fleximod/cli.py similarity index 81% rename from .lib/git_fleximod/cli.py rename to .lib/git-fleximod/git_fleximod/cli.py index 9169b338..b5a05494 100644 --- a/.lib/git_fleximod/cli.py +++ b/.lib/git-fleximod/git_fleximod/cli.py @@ -1,9 +1,8 @@ from pathlib import Path import argparse, os, sys -from importlib.resources import files from git_fleximod import utils -__version__ = "1.0.0" +__version__ = "1.0.2" class CustomArgumentParser(argparse.ArgumentParser): def print_help(self, file=None): @@ -12,8 +11,8 @@ def print_help(self, file=None): # Then append the contents of README.md candidate_paths = [ - os.path.join(sys.prefix, "share", "your-package", "README.md"), - os.path.join(os.path.dirname(__file__), "..", "README.md") # fallback for dev + Path(sys.prefix) / "share" / "git_fleximod" / "README.md", + Path(__file__).resolve().parent.parent / "README.md", # fallback for dev ] for path in candidate_paths: if os.path.exists(path): @@ -25,26 +24,21 @@ def print_help(self, file=None): def find_root_dir(filename=".gitmodules"): """ finds the highest directory in tree which contains a file called filename """ - try: - root = utils.execute_subprocess(["git","rev-parse", "--show-toplevel"], - output_to_caller=True ).rstrip() - except: - d = Path.cwd() - root = Path(d.root) - dirlist = [] - dl = d - while dl != root: - dirlist.append(dl) - dl = dl.parent - dirlist.append(root) - dirlist.reverse() - - for dl in dirlist: - attempt = dl / filename - if attempt.is_file(): - return str(dl) - return None - return Path(root) + d = Path.cwd() + root = Path(d.root) + dirlist = [] + dl = d + while dl != root: + dirlist.append(dl) + dl = dl.parent + dirlist.append(root) + dirlist.reverse() + + for dl in dirlist: + attempt = dl / filename + if attempt.is_file(): + return str(dl) + return None def get_parser(): description = """ diff --git a/.lib/git_fleximod/git_fleximod.py b/.lib/git-fleximod/git_fleximod/git_fleximod.py similarity index 100% rename from .lib/git_fleximod/git_fleximod.py rename to .lib/git-fleximod/git_fleximod/git_fleximod.py diff --git a/.lib/git_fleximod/gitinterface.py b/.lib/git-fleximod/git_fleximod/gitinterface.py similarity index 93% rename from .lib/git_fleximod/gitinterface.py rename to .lib/git-fleximod/git_fleximod/gitinterface.py index 1a736e4e..022426d2 100644 --- a/.lib/git_fleximod/gitinterface.py +++ b/.lib/git-fleximod/git_fleximod/gitinterface.py @@ -60,7 +60,14 @@ def _git_operation_command(self, operation, args): # pylint: disable=unused-argument def git_operation(self, operation, *args, **kwargs): - command = self._git_operation_command(operation, args) + newargs = [] + for a in args: + # Do not use ssh interface + if isinstance(a, str): + a = a.replace("git@github.com:", "https://github.com/") + newargs.append(a) + + command = self._git_command(operation, *newargs) if isinstance(command, list): try: status, output = utils.execute_subprocess(command, status_to_caller=True, output_to_caller=True) diff --git a/.lib/git_fleximod/gitmodules.py b/.lib/git-fleximod/git_fleximod/gitmodules.py similarity index 100% rename from .lib/git_fleximod/gitmodules.py rename to .lib/git-fleximod/git_fleximod/gitmodules.py diff --git a/.lib/git_fleximod/lstripreader.py b/.lib/git-fleximod/git_fleximod/lstripreader.py similarity index 100% rename from .lib/git_fleximod/lstripreader.py rename to .lib/git-fleximod/git_fleximod/lstripreader.py diff --git a/.lib/git_fleximod/metoflexi.py b/.lib/git-fleximod/git_fleximod/metoflexi.py similarity index 100% rename from .lib/git_fleximod/metoflexi.py rename to .lib/git-fleximod/git_fleximod/metoflexi.py diff --git a/.lib/git_fleximod/submodule.py b/.lib/git-fleximod/git_fleximod/submodule.py similarity index 88% rename from .lib/git_fleximod/submodule.py rename to .lib/git-fleximod/git_fleximod/submodule.py index 572237d0..ba07c117 100644 --- a/.lib/git_fleximod/submodule.py +++ b/.lib/git-fleximod/git_fleximod/submodule.py @@ -27,7 +27,7 @@ def __init__(self, root_dir, name, path, url, fxtag=None, fxurl=None, fxsparse=N """ self.name = name self.root_dir = root_dir - self.path = path + self.path = path self.url = url self.fxurl = fxurl self.fxtag = fxtag @@ -37,14 +37,14 @@ def __init__(self, root_dir, name, path, url, fxtag=None, fxurl=None, fxsparse=N else: self.fxrequired = "AlwaysRequired" self.logger = logger - + def status(self): """ Checks the status of the submodule and returns 4 parameters: - result (str): The status of the submodule. - needsupdate (bool): An indicator if the submodule needs to be updated. - localmods (bool): An indicator if the submodule has local modifications. - - testfails (bool): An indicator if the submodule has failed a test, this is used for testing purposes. + - testfails (bool): An indicator if the submodule has failed a test, this is used for testing purposes. """ smpath = os.path.join(self.root_dir, self.path) @@ -54,7 +54,7 @@ def status(self): ahash = None optional = "" if "Optional" in self.fxrequired: - optional = " (optional)" + optional = " (optional)" required = None level = None if not os.path.exists(os.path.join(smpath, ".git")): @@ -63,7 +63,7 @@ def status(self): status, tags = rootgit.git_operation("ls-remote", "--tags", self.url) status, result = rootgit.git_operation("submodule","status",smpath) result = result.split() - + if result: ahash = result[0][1:] hhash = None @@ -102,7 +102,7 @@ def status(self): result = f"e {self.name:>20} has no associated remote" testfails = True needsupdate = True - return result, needsupdate, localmods, testfails + return result, needsupdate, localmods, testfails status, rurl = git.git_operation("ls-remote","--get-url") status, lines = git.git_operation("log", "--pretty=format:\"%h %d\"") line = lines.partition('\n')[0] @@ -119,27 +119,25 @@ def status(self): atag = atag[:-1] if atag == self.fxtag: break - - - #print(f"line is {line} ahash is {ahash} atag is {atag} {parts}") - # atag = git.git_operation("describe", "--tags", "--always") - # ahash = git.git_operation("rev-list", "HEAD").partition("\n")[0] - recurse = False if rurl != self.url: remote = self._add_remote(git) git.git_operation("fetch", remote) + # Asked for a tag and found that tag if self.fxtag and atag == self.fxtag: result = f" {self.name:>20} at tag {self.fxtag}" recurse = True testfails = False + # Asked for and found a hash elif self.fxtag and (ahash[: len(self.fxtag)] == self.fxtag or (self.fxtag.find(ahash)==0)): result = f" {self.name:>20} at hash {ahash}" recurse = True testfails = False + # Asked for and found a hash elif atag == ahash: result = f" {self.name:>20} at hash {ahash}" recurse = True + # Did not find requested tag or hash elif self.fxtag: result = f"s {self.name:>20} {atag} {ahash} is out of sync with .gitmodules {self.fxtag}" testfails = True @@ -150,7 +148,7 @@ def status(self): else: result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {ahash}" testfails = False - + status, output = git.git_operation("status", "--ignore-submodules", "-uno") if "nothing to commit" not in output: localmods = True @@ -158,7 +156,7 @@ def status(self): # print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}") return result, needsupdate, localmods, testfails - + def _add_remote(self, git): """ Adds a new remote to the submodule if it does not already exist. @@ -172,7 +170,7 @@ def _add_remote(self, git): Returns: str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL. - """ + """ status, remotes = git.git_operation("remote", "-v") remotes = remotes.splitlines() upstream = None @@ -214,7 +212,7 @@ def sparse_checkout(self): Returns: None - """ + """ self.logger.info("Called sparse_checkout for {}".format(self.name)) rgit = GitInterface(self.root_dir, self.logger) status, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") @@ -252,7 +250,7 @@ def sparse_checkout(self): sprepo_git.config_set_value("core", "sparseCheckout", "true") # set the repository remote - + self.logger.info("Setting remote origin in {}/{}".format(self.root_dir, self.path)) status, remotes = sprepo_git.git_operation("remote", "-v") if self.url not in remotes: @@ -284,17 +282,18 @@ def sparse_checkout(self): if not os.path.isdir(infodir): os.makedirs(infodir) gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout")) - if os.path.isfile(gitsparse): - self.logger.warning( - "submodule {} is already initialized {}".format(self.name, rootdotgit) - ) - return + if os.path.isfile(gitsparse): + self.logger.warning( + "submodule {} is already initialized {}".format(self.name, rootdotgit) + ) + os.remove(gitsparse) - with utils.pushd(sprep_repo): if os.path.isfile(self.fxsparse): - shutil.copy(self.fxsparse, gitsparse) - + else: + self.logger.warning( + "submodule {} could not find {}".format(self.name, self.fxsparse) + ) # Finally checkout the repo sprepo_git.git_operation("fetch", "origin", "--tags") @@ -303,6 +302,13 @@ def sparse_checkout(self): print(f"Error checking out {self.name:>20} at {self.fxtag}") else: print(f"Successfully checked out {self.name:>20} at {self.fxtag}") + status,f = sprepo_git.git_operation("status") + # Restore any files deleted from sandbox + for line in f.splitlines(): + if "deleted:" in line: + deleted_file = line.split("deleted:")[1].strip() + sprepo_git.git_operation("checkout", deleted_file) + rgit.config_set_value('submodule.' + self.name, "active", "true") rgit.config_set_value('submodule.' + self.name, "url", self.url) rgit.config_set_value('submodule.' + self.name, "path", self.path) @@ -341,6 +347,9 @@ async def update(self): # Look for a .gitmodules file in the newly checkedout repo if self.fxsparse: print(f"Sparse checkout {self.name} fxsparse {self.fxsparse}") + if not os.path.isfile(self.fxsparse): + self.logger.info("Submodule {} fxsparse file not found".format(self.name)) + self.sparse_checkout() else: if not repo_exists and self.url: @@ -367,7 +376,7 @@ async def update(self): shutil.rmtree(os.path.join(repodir, ".git")) else: shutil.move(os.path.join(repodir, ".git"), newpath) - + with open(os.path.join(repodir, ".git"), "w") as f: f.write("gitdir: " + os.path.relpath(newpath, start=repodir)) @@ -375,28 +384,34 @@ async def update(self): parent = os.path.dirname(repodir) if not os.path.isdir(parent): os.makedirs(parent) - git.git_operation("submodule", "add", "--name", self.name, "--", self.url, self.path) + git.git_operation("submodule", "add", "--name", self.name, "--", self.url, self.path) if not repo_exists: git.git_operation("submodule", "init", "--", self.path) await git.git_operation_async("submodule", "update", "--", self.path) - if self.fxtag: + if self.fxtag: smgit = GitInterface(repodir, self.logger) newremote = self._add_remote(smgit) # Trying to distingush a tag from a hash - allowed = set(string.digits + 'abcdef') + allowed = set(string.digits + 'abcdef') + status = 0 if not set(self.fxtag) <= allowed: # This is a tag tag = f"refs/tags/{self.fxtag}:refs/tags/{self.fxtag}" - smgit.git_operation("fetch", newremote, tag) - smgit.git_operation("checkout", self.fxtag) + status,_ = smgit.git_operation("fetch", newremote, tag) + if status == 0: + status,_ = smgit.git_operation("checkout", self.fxtag) + if status: + utils.fatal_error( + f"Failed to checkout {self.name} at tag or hash {self.fxtag} from {repodir}" + ) if not os.path.exists(os.path.join(repodir, ".git")): utils.fatal_error( f"Failed to checkout {self.name} {repo_exists} {repodir} {self.path}" ) - + if os.path.exists(os.path.join(self.path, ".git")): submoddir = os.path.join(self.root_dir, self.path) @@ -409,6 +424,18 @@ async def update(self): if fxtag and fxtag not in tags: git.git_operation("fetch", newremote, "--tags") status, atag = git.git_operation("describe", "--tags", "--always") + status, files = git.git_operation("diff", "--name-only", "-z") + modfiles = [] + moddirs = [] + if files: + for f in files.split('\0'): + if f: + if os.path.exists(f): + git.git_operation("checkout",f) + elif os.path.isdir(f): + moddirs.append(f) + else: + modfiles.append(f) if fxtag and fxtag != atag: try: status, _ = git.git_operation("checkout", fxtag) @@ -416,13 +443,17 @@ async def update(self): print(f"{self.name:>20} updated to {fxtag}") except Exception as error: print(error) - + elif not fxtag: print(f"No fxtag found for submodule {self.name:>20}") + elif modfiles: + print(f"{self.name:>20} has modified files: {modfiles}") + elif moddirs: + print(f"{self.name:>20} has modified directories: {moddirs}") else: print(f"{self.name:>20} up to date.") - + return diff --git a/.lib/git_fleximod/utils.py b/.lib/git-fleximod/git_fleximod/utils.py similarity index 100% rename from .lib/git_fleximod/utils.py rename to .lib/git-fleximod/git_fleximod/utils.py diff --git a/.lib/git-fleximod/poetry.lock b/.lib/git-fleximod/poetry.lock new file mode 100644 index 00000000..ac82fb0d --- /dev/null +++ b/.lib/git-fleximod/poetry.lock @@ -0,0 +1,693 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "alabaster" +version = "0.7.13" +description = "A configurable sidebar-enabled Sphinx theme" +optional = false +python-versions = ">=3.6" +files = [ + {file = "alabaster-0.7.13-py3-none-any.whl", hash = "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3"}, + {file = "alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"}, +] + +[[package]] +name = "babel" +version = "2.15.0" +description = "Internationalization utilities" +optional = false +python-versions = ">=3.8" +files = [ + {file = "Babel-2.15.0-py3-none-any.whl", hash = "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb"}, + {file = "babel-2.15.0.tar.gz", hash = "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"}, +] + +[package.dependencies] +pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} + +[package.extras] +dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] + +[[package]] +name = "certifi" +version = "2024.8.30" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "docutils" +version = "0.19" +description = "Docutils -- Python Documentation Utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, + {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.1" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "fsspec" +version = "2023.12.2" +description = "File-system specification" +optional = false +python-versions = ">=3.8" +files = [ + {file = "fsspec-2023.12.2-py3-none-any.whl", hash = "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"}, + {file = "fsspec-2023.12.2.tar.gz", hash = "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb"}, +] + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +devel = ["pytest", "pytest-cov"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + +[[package]] +name = "gitdb" +version = "4.0.11" +description = "Git Object Database" +optional = false +python-versions = ">=3.7" +files = [ + {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, + {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, +] + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.43" +description = "GitPython is a Python library used to interact with Git repositories" +optional = false +python-versions = ">=3.7" +files = [ + {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, + {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, +] + +[package.dependencies] +gitdb = ">=4.0.1,<5" + +[package.extras] +doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] + +[[package]] +name = "idna" +version = "3.7" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, +] + +[[package]] +name = "imagesize" +version = "1.4.1" +description = "Getting image size from png/jpeg/jpeg2000/gif file" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, + {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, +] + +[[package]] +name = "importlib-metadata" +version = "8.0.0" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_metadata-8.0.0-py3-none-any.whl", hash = "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f"}, + {file = "importlib_metadata-8.0.0.tar.gz", hash = "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812"}, +] + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "jinja2" +version = "3.1.4" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, + {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "markupsafe" +version = "2.1.5" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] + +[[package]] +name = "packaging" +version = "24.1" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pyfakefs" +version = "5.5.0" +description = "pyfakefs implements a fake file system that mocks the Python file system modules." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyfakefs-5.5.0-py3-none-any.whl", hash = "sha256:8dbf203ab7bef1529f11f7d41b9478b898e95bf9f3b71262163aac07a518cd76"}, + {file = "pyfakefs-5.5.0.tar.gz", hash = "sha256:7448aaa07142f892d0a4eb52a5ed3206a9f02c6599e686cd97d624c18979c154"}, +] + +[[package]] +name = "pygments" +version = "2.18.0" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, + {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pytest" +version = "8.2.2" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, + {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2.0" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytz" +version = "2024.1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "smmap" +version = "5.0.1" +description = "A pure Python implementation of a sliding window memory map manager" +optional = false +python-versions = ">=3.7" +files = [ + {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, + {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, +] + +[[package]] +name = "snowballstemmer" +version = "2.2.0" +description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." +optional = false +python-versions = "*" +files = [ + {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, + {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, +] + +[[package]] +name = "sphinx" +version = "5.3.0" +description = "Python documentation generator" +optional = false +python-versions = ">=3.6" +files = [ + {file = "Sphinx-5.3.0.tar.gz", hash = "sha256:51026de0a9ff9fc13c05d74913ad66047e104f56a129ff73e174eb5c3ee794b5"}, + {file = "sphinx-5.3.0-py3-none-any.whl", hash = "sha256:060ca5c9f7ba57a08a1219e547b269fadf125ae25b06b9fa7f66768efb652d6d"}, +] + +[package.dependencies] +alabaster = ">=0.7,<0.8" +babel = ">=2.9" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +docutils = ">=0.14,<0.20" +imagesize = ">=1.3" +importlib-metadata = {version = ">=4.8", markers = "python_version < \"3.10\""} +Jinja2 = ">=3.0" +packaging = ">=21.0" +Pygments = ">=2.12" +requests = ">=2.5.0" +snowballstemmer = ">=2.0" +sphinxcontrib-applehelp = "*" +sphinxcontrib-devhelp = "*" +sphinxcontrib-htmlhelp = ">=2.0.0" +sphinxcontrib-jsmath = "*" +sphinxcontrib-qthelp = "*" +sphinxcontrib-serializinghtml = ">=1.1.5" + +[package.extras] +docs = ["sphinxcontrib-websupport"] +lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-bugbear", "flake8-comprehensions", "flake8-simplify", "isort", "mypy (>=0.981)", "sphinx-lint", "types-requests", "types-typed-ast"] +test = ["cython", "html5lib", "pytest (>=4.6)", "typed_ast"] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "1.0.4" +description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" +optional = false +python-versions = ">=3.8" +files = [ + {file = "sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e"}, + {file = "sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "1.0.2" +description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." +optional = false +python-versions = ">=3.5" +files = [ + {file = "sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"}, + {file = "sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.0.1" +description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff"}, + {file = "sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["html5lib", "pytest"] + +[[package]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +description = "A sphinx extension which renders display math in HTML via JavaScript" +optional = false +python-versions = ">=3.5" +files = [ + {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, + {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, +] + +[package.extras] +test = ["flake8", "mypy", "pytest"] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "1.0.3" +description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." +optional = false +python-versions = ">=3.5" +files = [ + {file = "sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72"}, + {file = "sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "1.1.5" +description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." +optional = false +python-versions = ">=3.5" +files = [ + {file = "sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952"}, + {file = "sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd"}, +] + +[package.extras] +lint = ["docutils-stubs", "flake8", "mypy"] +test = ["pytest"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "urllib3" +version = "2.2.2" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wheel" +version = "0.42.0" +description = "A built-package format for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "wheel-0.42.0-py3-none-any.whl", hash = "sha256:177f9c9b0d45c47873b619f5b650346d632cdc35fb5e4d25058e09c9e581433d"}, + {file = "wheel-0.42.0.tar.gz", hash = "sha256:c45be39f7882c9d34243236f2d63cbd58039e360f85d0913425fbd7ceea617a8"}, +] + +[package.extras] +test = ["pytest (>=6.0.0)", "setuptools (>=65)"] + +[[package]] +name = "zipp" +version = "3.19.2" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, + {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, +] + +[package.extras] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] + +[metadata] +lock-version = "2.0" +python-versions = "^3.8" +content-hash = "25ee2ae1d74abedde3a6637a60d4a3095ea5cf9731960875741bbc2ba84a475d" diff --git a/.lib/git-fleximod/pyproject.toml b/.lib/git-fleximod/pyproject.toml new file mode 100644 index 00000000..029fd65e --- /dev/null +++ b/.lib/git-fleximod/pyproject.toml @@ -0,0 +1,42 @@ +[tool.poetry] +name = "git-fleximod" +version = "1.0.2" +description = "Extended support for git-submodule and git-sparse-checkout" +authors = ["Jim Edwards "] +maintainers = ["Jim Edwards "] +license = "MIT" +readme = "README.md" +homepage = "https://github.com/jedwards4b/git-fleximod" +keywords = ["git", "submodule", "sparse-checkout"] +packages = [ +{ include = "git_fleximod"}, +{ include = "doc"}, +{ include = "README.md"}, +] + +[tool.poetry.scripts] +git-fleximod = "git_fleximod.git_fleximod:main" +me2flexi = "git_fleximod.metoflexi:_main" +fsspec = "fsspec.fuse:main" + +[tool.poetry.dependencies] +python = "^3.8" +GitPython = "^3.1.0" +sphinx = "^5.0.0" +fsspec = "^2023.12.2" +wheel = "^0.42.0" +pytest = "^8.0.0" +pyfakefs = "^5.3.5" + +[tool.poetry.urls] +"Bug Tracker" = "https://github.com/jedwards4b/git-fleximod/issues" + +[tool.pytest.ini_options] +markers = [ + "skip_after_first: only run on first iteration" +] + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + diff --git a/.lib/git-fleximod/tbump.toml b/.lib/git-fleximod/tbump.toml new file mode 100644 index 00000000..f6fe1e88 --- /dev/null +++ b/.lib/git-fleximod/tbump.toml @@ -0,0 +1,43 @@ +# Uncomment this if your project is hosted on GitHub: +github_url = "https://github.com/jedwards4b/git-fleximod/" + +[version] +current = "1.0.2" + +# Example of a semver regexp. +# Make sure this matches current_version before +# using tbump +regex = ''' + (?P\d+) + \. + (?P\d+) + \. + (?P\d+) + ''' + +[git] +message_template = "Bump to {new_version}" +tag_template = "v{new_version}" + +# For each file to patch, add a [[file]] config +# section containing the path of the file, relative to the +# tbump.toml location. +[[file]] +src = "git_fleximod/cli.py" + +[[file]] +src = "pyproject.toml" + +# You can specify a list of commands to +# run after the files have been patched +# and before the git commit is made + +# [[before_commit]] +# name = "check changelog" +# cmd = "grep -q {new_version} Changelog.rst" + +# Or run some commands after the git tag and the branch +# have been pushed: +# [[after_push]] +# name = "publish" +# cmd = "./publish.sh" diff --git a/.lib/git-fleximod/tests/__init__.py b/.lib/git-fleximod/tests/__init__.py new file mode 100644 index 00000000..4d4c66c7 --- /dev/null +++ b/.lib/git-fleximod/tests/__init__.py @@ -0,0 +1,3 @@ +import sys, os + +sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir, "src")) diff --git a/.lib/git-fleximod/tests/conftest.py b/.lib/git-fleximod/tests/conftest.py new file mode 100644 index 00000000..44d28e17 --- /dev/null +++ b/.lib/git-fleximod/tests/conftest.py @@ -0,0 +1,150 @@ +import pytest +from git_fleximod.gitinterface import GitInterface +import os +import subprocess +import logging +from pathlib import Path + +@pytest.fixture(scope='session') +def logger(): + logging.basicConfig( + level=logging.INFO, format="%(name)s - %(levelname)s - %(message)s", handlers=[logging.StreamHandler()] + ) + logger = logging.getLogger(__name__) + return logger + +all_repos=[ + {"subrepo_path": "modules/test", + "submodule_name": "test_submodule", + "status1" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", + "status2" : "test_submodule at tag MPIserial_2.4.0", + "status3" : "test_submodule at tag MPIserial_2.4.0", + "status4" : "test_submodule at tag MPIserial_2.4.0", + "gitmodules_content" : """ + [submodule "test_submodule"] + path = modules/test + url = https://github.com/ESMCI/mpi-serial.git + fxtag = MPIserial_2.4.0 + fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = ToplevelRequired +"""}, + {"subrepo_path": "modules/test_optional", + "submodule_name": "test_optional", + "status1" : "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", + "status2" : "test_optional at tag MPIserial_2.4.0", + "status3" : "test_optional not checked out, out of sync at tag MPIserial_2.5.4, expected tag is MPIserial_2.4.0 (optional)", + "status4" : "test_optional at tag MPIserial_2.4.0", + "gitmodules_content": """ + [submodule "test_optional"] + path = modules/test_optional + url = https://github.com/ESMCI/mpi-serial.git + fxtag = MPIserial_2.4.0 + fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = ToplevelOptional +"""}, + {"subrepo_path": "modules/test_alwaysoptional", + "submodule_name": "test_alwaysoptional", + "status1" : "test_alwaysoptional MPIserial_2.3.0 is out of sync with .gitmodules e5cf35c", + "status2" : "test_alwaysoptional at hash e5cf35c", + "status3" : "out of sync at tag MPIserial_2.5.4, expected tag is e5cf35c", + "status4" : "test_alwaysoptional at hash e5cf35c", + "gitmodules_content": """ + [submodule "test_alwaysoptional"] + path = modules/test_alwaysoptional + url = https://github.com/ESMCI/mpi-serial.git + fxtag = e5cf35c + fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = AlwaysOptional +"""}, + {"subrepo_path": "modules/test_sparse", + "submodule_name": "test_sparse", + "status1" : "test_sparse at tag MPIserial_2.5.0", + "status2" : "test_sparse at tag MPIserial_2.5.0", + "status3" : "test_sparse at tag MPIserial_2.5.0", + "status4" : "test_sparse at tag MPIserial_2.5.0", + "gitmodules_content": """ + [submodule "test_sparse"] + path = modules/test_sparse + url = https://github.com/ESMCI/mpi-serial.git + fxtag = MPIserial_2.5.0 + fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = AlwaysRequired + fxsparse = ../.sparse_file_list +"""}, +] +@pytest.fixture(params=all_repos) + +def shared_repos(request): + return request.param + +@pytest.fixture +def get_all_repos(): + return all_repos + +def write_sparse_checkout_file(fp): + sparse_content = """m4 +""" + fp.write_text(sparse_content) + +@pytest.fixture +def test_repo(shared_repos, tmp_path, logger): + subrepo_path = shared_repos["subrepo_path"] + submodule_name = shared_repos["submodule_name"] + test_dir = tmp_path / "testrepo" + test_dir.mkdir() + str_path = str(test_dir) + gitp = GitInterface(str_path, logger) + assert test_dir.joinpath(".git").is_dir() + (test_dir / "modules").mkdir() + if "sparse" in submodule_name: + (test_dir / subrepo_path).mkdir() + # Add the sparse checkout file + write_sparse_checkout_file(test_dir / "modules" / ".sparse_file_list") + gitp.git_operation("add","modules/.sparse_file_list") + else: + gitp = GitInterface(str(test_dir), logger) + gitp.git_operation("submodule", "add", "--depth","1","--name", submodule_name, "https://github.com/ESMCI/mpi-serial.git", subrepo_path) + assert test_dir.joinpath(".gitmodules").is_file() + gitp.git_operation("add",subrepo_path) + gitp.git_operation("commit","-a","-m","\"add submod\"") + test_dir2 = tmp_path / "testrepo2" + gitp.git_operation("clone",test_dir,test_dir2) + return test_dir2 + + +@pytest.fixture +def complex_repo(tmp_path, logger): + test_dir = tmp_path / "testcomplex" + test_dir.mkdir() + str_path = str(test_dir) + gitp = GitInterface(str_path, logger) + gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") + gitp.git_operation("fetch", "origin") + gitp.git_operation("checkout", "v0.0.1") + return test_dir + +@pytest.fixture +def complex_update(tmp_path, logger): + test_dir = tmp_path / "testcomplex" + test_dir.mkdir() + str_path = str(test_dir) + gitp = GitInterface(str_path, logger) + gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") + gitp.git_operation("fetch", "origin") + gitp.git_operation("checkout", "v0.0.2") + + return test_dir + +@pytest.fixture +def git_fleximod(): + def _run_fleximod(path, args, input=None): + cmd = ["git", "fleximod"] + args.split() + result = subprocess.run(cmd, cwd=path, input=input, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True) + if result.returncode: + print(result.stdout) + print(result.stderr) + return result + return _run_fleximod + diff --git a/.lib/git-fleximod/tests/test_a_import.py b/.lib/git-fleximod/tests/test_a_import.py new file mode 100644 index 00000000..d5ca878d --- /dev/null +++ b/.lib/git-fleximod/tests/test_a_import.py @@ -0,0 +1,8 @@ +# pylint: disable=unused-import +from git_fleximod import cli +from git_fleximod import utils +from git_fleximod.gitinterface import GitInterface +from git_fleximod.gitmodules import GitModules + +def test_import(): + print("here") diff --git a/.lib/git-fleximod/tests/test_b_update.py b/.lib/git-fleximod/tests/test_b_update.py new file mode 100644 index 00000000..159f1cfa --- /dev/null +++ b/.lib/git-fleximod/tests/test_b_update.py @@ -0,0 +1,26 @@ +import pytest +from pathlib import Path + +def test_basic_checkout(git_fleximod, test_repo, shared_repos): + # Prepare a simple .gitmodules + gm = shared_repos['gitmodules_content'] + file_path = (test_repo / ".gitmodules") + repo_name = shared_repos["submodule_name"] + repo_path = shared_repos["subrepo_path"] + + file_path.write_text(gm) + + # Run the command + result = git_fleximod(test_repo, f"update {repo_name}") + + # Assertions + assert result.returncode == 0 + assert Path(test_repo / repo_path).exists() # Did the submodule directory get created? + if "sparse" in repo_name: + assert Path(test_repo / f"{repo_path}/m4").exists() # Did the submodule sparse directory get created? + assert not Path(test_repo / f"{repo_path}/README").exists() # Did only the submodule sparse directory get created? + + status = git_fleximod(test_repo, f"status {repo_name}") + + assert shared_repos["status2"] in status.stdout + diff --git a/.lib/git-fleximod/tests/test_c_required.py b/.lib/git-fleximod/tests/test_c_required.py new file mode 100644 index 00000000..2ac66145 --- /dev/null +++ b/.lib/git-fleximod/tests/test_c_required.py @@ -0,0 +1,43 @@ +import pytest +import re +from pathlib import Path + +def test_required(git_fleximod, test_repo, shared_repos): + file_path = (test_repo / ".gitmodules") + gm = shared_repos["gitmodules_content"] + repo_name = shared_repos["submodule_name"] + if file_path.exists(): + with file_path.open("r") as f: + gitmodules_content = f.read() + # add the entry if it does not exist + if repo_name not in gitmodules_content: + file_path.write_text(gitmodules_content+gm) + # or if it is incomplete + elif gm not in gitmodules_content: + file_path.write_text(gm) + else: + file_path.write_text(gm) + result = git_fleximod(test_repo, "update") + assert result.returncode == 0 + status = git_fleximod(test_repo, f"status {repo_name}") + assert shared_repos["status3"] in status.stdout + status = git_fleximod(test_repo, f"update --optional") + assert result.returncode == 0 + status = git_fleximod(test_repo, f"status {repo_name}") + assert shared_repos["status4"] in status.stdout + status = git_fleximod(test_repo, f"update {repo_name}") + assert result.returncode == 0 + status = git_fleximod(test_repo, f"status {repo_name}") + assert shared_repos["status4"] in status.stdout + + text = file_path.read_text() + new_value = "somethingelse" + pattern = r"(^\s*fxtag\s*=\s*).*$" + replacement = r"\1" + new_value + new_text = re.sub(pattern, replacement, text, flags=re.MULTILINE) + + # Write updated content back to file + file_path.write_text(new_text) + + result = git_fleximod(test_repo, f"update {repo_name}") + assert f'fatal: couldn\'t find remote ref' in result.stderr or 'error: pathspec \'somethingelse\' did not match any file(s) known to git' in result.stderr diff --git a/.lib/git-fleximod/tests/test_d_complex.py b/.lib/git-fleximod/tests/test_d_complex.py new file mode 100644 index 00000000..edde7d81 --- /dev/null +++ b/.lib/git-fleximod/tests/test_d_complex.py @@ -0,0 +1,66 @@ +import pytest +from pathlib import Path +from git_fleximod.gitinterface import GitInterface + +def test_complex_checkout(git_fleximod, complex_repo, logger): + status = git_fleximod(complex_repo, "status") + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout) + assert("Complex not checked out, aligned at tag testtag02" in status.stdout) + assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) + + # This should checkout and update test_submodule and complex_sub + result = git_fleximod(complex_repo, "update") + assert result.returncode == 0 + + status = git_fleximod(complex_repo, "status") + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag02" in status.stdout) + + # now check the complex_sub + root = (complex_repo / "modules" / "complex") + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serial" / ".git").exists()) + assert(not (root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + # update a single optional submodule + + result = git_fleximod(complex_repo, "update ToplevelOptional") + assert result.returncode == 0 + + status = git_fleximod(complex_repo, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag02" in status.stdout) + assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) + + # Finally update optional + result = git_fleximod(complex_repo, "update --optional") + assert result.returncode == 0 + + status = git_fleximod(complex_repo, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag02" in status.stdout) + assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout) + + # now check the complex_sub + root = (complex_repo / "modules" / "complex" ) + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + diff --git a/.lib/git-fleximod/tests/test_e_complex_update.py b/.lib/git-fleximod/tests/test_e_complex_update.py new file mode 100644 index 00000000..0c3ab4c6 --- /dev/null +++ b/.lib/git-fleximod/tests/test_e_complex_update.py @@ -0,0 +1,69 @@ +import pytest +from pathlib import Path +from git_fleximod.gitinterface import GitInterface + +def test_complex_update(git_fleximod, complex_update, logger): + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout) + assert("Complex not checked out, out of sync at tag testtag02, expected tag is testtag3" in status.stdout) + assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) + + # This should checkout and update test_submodule and complex_sub + result = git_fleximod(complex_update, "update") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag3" in status.stdout) + + # now check the complex_sub + root = (complex_update / "modules" / "complex") + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serialAR" / ".git").exists()) + assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) + assert(not (root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + # update a single optional submodule + + result = git_fleximod(complex_update, "update ToplevelOptional") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag3" in status.stdout) + assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) + + # Finally update optional + result = git_fleximod(complex_update, "update --optional") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag3" in status.stdout) + assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout) + + # now check the complex_sub + root = (complex_update / "modules" / "complex" ) + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert(not (root / "modules" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serialAR" / ".git").exists()) + assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + From f1827392effca94894e0e1f00365fca76c1b808b Mon Sep 17 00:00:00 2001 From: peverwhee Date: Mon, 13 Oct 2025 20:48:46 -0600 Subject: [PATCH 098/150] code cleanup --- src/dynamics/mpas/dyn_comp.F90 | 2 +- src/dynamics/mpas/dyn_comp_impl.F90 | 2 +- src/dynamics/se/dyn_comp.F90 | 2 +- src/physics/utils/orbital_data.F90 | 6 ------ src/utils/machine.F90 | 2 +- test/unit/fortran/src/pio_reader/test_pio_reader.pf | 2 +- 6 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/dynamics/mpas/dyn_comp.F90 b/src/dynamics/mpas/dyn_comp.F90 index d5f1c9d9..76e50e37 100644 --- a/src/dynamics/mpas/dyn_comp.F90 +++ b/src/dynamics/mpas/dyn_comp.F90 @@ -47,7 +47,7 @@ end subroutine dyn_readnl module subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) use runtime_obj, only: runtime_options - type(runtime_options), intent(in) :: cam_runtime_opts + type(runtime_options), intent(in) :: cam_runtime_opts type(dyn_import_t), intent(in) :: dyn_in type(dyn_export_t), intent(in) :: dyn_out end subroutine dyn_init diff --git a/src/dynamics/mpas/dyn_comp_impl.F90 b/src/dynamics/mpas/dyn_comp_impl.F90 index 46d7931a..08a7c24f 100644 --- a/src/dynamics/mpas/dyn_comp_impl.F90 +++ b/src/dynamics/mpas/dyn_comp_impl.F90 @@ -142,7 +142,7 @@ module subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) ! Module(s) from external libraries. use pio, only: file_desc_t - type(runtime_options), intent(in) :: cam_runtime_opts + type(runtime_options), intent(in) :: cam_runtime_opts type(dyn_import_t), intent(in) :: dyn_in type(dyn_export_t), intent(in) :: dyn_out diff --git a/src/dynamics/se/dyn_comp.F90 b/src/dynamics/se/dyn_comp.F90 index a35188e9..5acbf89c 100644 --- a/src/dynamics/se/dyn_comp.F90 +++ b/src/dynamics/se/dyn_comp.F90 @@ -556,7 +556,7 @@ end subroutine dyn_readnl !========================================================================================= -subroutine dyn_init(dyn_in, dyn_out) +subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) use runtime_obj, only: runtime_options, set_cam_dycore use dyn_grid, only: elem, fvm use cam_pio_utils, only: clean_iodesc_list diff --git a/src/physics/utils/orbital_data.F90 b/src/physics/utils/orbital_data.F90 index 393f5247..c96d6459 100644 --- a/src/physics/utils/orbital_data.F90 +++ b/src/physics/utils/orbital_data.F90 @@ -23,7 +23,6 @@ module orbital_data real(kind_phys), protected, public :: solar_declination = FILL_R8 ! Solar declination angle [radians] real(kind_phys), protected, public :: earth_sun_distance = FILL_R8 ! Earth-sun distance [AU] real(kind_phys), allocatable, protected, public :: solar_zenith_angle(:) ! Solar zenith angle (column) [radians] - !real(kind_phys), allocatable, protected, public :: coszrs_rad(:) ! Cosine of solar zenith angle (column) for radiation [radians] ! Local parameters character(len=*), parameter :: module_name = '(orbital_data)' @@ -53,11 +52,6 @@ subroutine orbital_data_init(number_of_columns) call check_allocate(error_code, subroutine_name, & 'solar_zenith_angle(number_of_columns)', & file=__FILE__, line=__LINE__) - !allocate(coszrs_rad(number_of_columns), source=FILL_R8, & - ! stat=error_code) - !call check_allocate(error_code, subroutine_name, & - ! 'coszrs_rad(number_of_columns)', & - ! file=__FILE__, line=__LINE__) end subroutine orbital_data_init diff --git a/src/utils/machine.F90 b/src/utils/machine.F90 index 4d1a37e4..8d186ad8 100644 --- a/src/utils/machine.F90 +++ b/src/utils/machine.F90 @@ -1,4 +1,4 @@ -! This module is the CAM version of the CCPP generated module of the same name +! This module is the CAM-SIMA version of the CCPP generated module of the same name module machine use ccpp_kinds, only: kind_phys => kind_phys diff --git a/test/unit/fortran/src/pio_reader/test_pio_reader.pf b/test/unit/fortran/src/pio_reader/test_pio_reader.pf index a34a64e3..accb7f0c 100644 --- a/test/unit/fortran/src/pio_reader/test_pio_reader.pf +++ b/test/unit/fortran/src/pio_reader/test_pio_reader.pf @@ -1930,7 +1930,7 @@ subroutine test_pio_reader_bad_count_element_range() @assertNotEqual(0, errcode) @assertEqual(expected_err_msg_neg, trim(errmsg)) - ! Next attempt to read variable with too-large start element: + ! Next attempt to read variable with too-large count element: call reader%get_var("pressure", pressure, errmsg, errcode, start=(/1/), count=(/60/)) ! Check that the correct error was raised: From 92f7b763cb577a07d74e0ed1c4228dd8ef2dffec Mon Sep 17 00:00:00 2001 From: peverwhee Date: Mon, 13 Oct 2025 20:54:07 -0600 Subject: [PATCH 099/150] remove final redundant git-fleximod files --- .lib/.github/workflows/pre-commit | 13 ----- .lib/.github/workflows/pytest.yaml | 77 ------------------------------ .lib/.pre-commit-config.yaml | 18 ------- 3 files changed, 108 deletions(-) delete mode 100644 .lib/.github/workflows/pre-commit delete mode 100644 .lib/.github/workflows/pytest.yaml delete mode 100644 .lib/.pre-commit-config.yaml diff --git a/.lib/.github/workflows/pre-commit b/.lib/.github/workflows/pre-commit deleted file mode 100644 index 1a6ad008..00000000 --- a/.lib/.github/workflows/pre-commit +++ /dev/null @@ -1,13 +0,0 @@ -name: pre-commit -on: - pull_request: - push: - branches: [main] - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: pre-commit/action@v3.0.0 diff --git a/.lib/.github/workflows/pytest.yaml b/.lib/.github/workflows/pytest.yaml deleted file mode 100644 index 0868dd9a..00000000 --- a/.lib/.github/workflows/pytest.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# Run this job on pushes to `main`, and for pull requests. If you don't specify -# `branches: [main], then this actions runs _twice_ on pull requests, which is -# annoying. - -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - # If you wanted to use multiple Python versions, you'd have specify a matrix in the job and - # reference the matrixe python version here. - - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - # Cache the installation of Poetry itself, e.g. the next step. This prevents the workflow - # from installing Poetry every time, which can be slow. Note the use of the Poetry version - # number in the cache key, and the "-0" suffix: this allows you to invalidate the cache - # manually if/when you want to upgrade Poetry, or if something goes wrong. This could be - # mildly cleaner by using an environment variable, but I don't really care. - - name: cache poetry install - uses: actions/cache@v4 - with: - path: ~/.local - key: poetry-1.7.1 - - # Install Poetry. You could do this manually, or there are several actions that do this. - # `snok/install-poetry` seems to be minimal yet complete, and really just calls out to - # Poetry's default install script, which feels correct. I pin the Poetry version here - # because Poetry does occasionally change APIs between versions and I don't want my - # actions to break if it does. - # - # The key configuration value here is `virtualenvs-in-project: true`: this creates the - # venv as a `.venv` in your testing directory, which allows the next step to easily - # cache it. - - uses: snok/install-poetry@v1 - with: - version: 1.7.1 - virtualenvs-create: true - virtualenvs-in-project: true - - # Cache your dependencies (i.e. all the stuff in your `pyproject.toml`). Note the cache - # key: if you're using multiple Python versions, or multiple OSes, you'd need to include - # them in the cache key. I'm not, so it can be simple and just depend on the poetry.lock. - - name: cache deps - id: cache-deps - uses: actions/cache@v4 - with: - path: .venv - key: pydeps-${{ hashFiles('**/poetry.lock') }} - - # Install dependencies. `--no-root` means "install all dependencies but not the project - # itself", which is what you want to avoid caching _your_ code. The `if` statement - # ensures this only runs on a cache miss. - - run: poetry install --no-interaction --no-root - if: steps.cache-deps.outputs.cache-hit != 'true' - - # Now install _your_ project. This isn't necessary for many types of projects -- particularly - # things like Django apps don't need this. But it's a good idea since it fully-exercises the - # pyproject.toml and makes that if you add things like console-scripts at some point that - # they'll be installed and working. - - run: poetry install --no-interaction - - # And finally run tests. I'm using pytest and all my pytest config is in my `pyproject.toml` - # so this line is super-simple. But it could be as complex as you need. - - run: | - git config --global user.name "${GITHUB_ACTOR}" - git config --global user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" - poetry run pytest - diff --git a/.lib/.pre-commit-config.yaml b/.lib/.pre-commit-config.yaml deleted file mode 100644 index 2f6089da..00000000 --- a/.lib/.pre-commit-config.yaml +++ /dev/null @@ -1,18 +0,0 @@ -exclude: ^utils/.*$ - -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 - hooks: - - id: end-of-file-fixer - - id: trailing-whitespace - - repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black - - repo: https://github.com/PyCQA/pylint - rev: v2.11.1 - hooks: - - id: pylint - args: - - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error,no-member From 01133f1a3d446248aa1af09f20105bc632bed57b Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Tue, 14 Oct 2025 13:50:49 -0400 Subject: [PATCH 100/150] Add gravity wave CAM4 and CAM7 SDF tests --- cime_config/testdefs/testlist_cam.xml | 18 +++++++ .../cam/outfrq_gw_cam4_derecho/shell_commands | 1 + .../cam/outfrq_gw_cam4_derecho/user_nl_cam | 28 ++++++++++ .../cam/outfrq_gw_cam7_derecho/shell_commands | 1 + .../cam/outfrq_gw_cam7_derecho/user_nl_cam | 51 +++++++++++++++++++ 5 files changed, 99 insertions(+) create mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam4_derecho/shell_commands create mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam4_derecho/user_nl_cam create mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam7_derecho/shell_commands create mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam7_derecho/user_nl_cam diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index da0b7dad..b3e02199 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -88,6 +88,24 @@ + + + + + + + + + + + + + + + + + + diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam4_derecho/shell_commands b/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam4_derecho/shell_commands new file mode 100644 index 00000000..0d4b3aa4 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam4_derecho/shell_commands @@ -0,0 +1 @@ + ./xmlchange CAM_CONFIG_OPTS="--dyn none --physics-suites gw_cam4" diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam4_derecho/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam4_derecho/user_nl_cam new file mode 100644 index 00000000..def1e644 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam4_derecho/user_nl_cam @@ -0,0 +1,28 @@ +! cam4 snapshot +pver = 26 +ncdata = '/glade/campaign/cesm/community/amwg/sima_baselines/cam_sima_test_snapshots/cam_ne3pg3_fhistc4_gw_drag_cam4_oro_snapshot_derecho_gnu_before_c20250826.nc' +ncdata_check = '/glade/campaign/cesm/community/amwg/sima_baselines/cam_sima_test_snapshots/cam_ne3pg3_fhistc4_gw_drag_cam4_oro_snapshot_derecho_gnu_after_c20250826.nc' + +debug_output = 0 + +! tolerances for testing +ncdata_check_err = .true. +min_difference = 2e-15 + +effgw_oro = 0.125D0 +gw_apply_tndmax = .true. +gw_dc = 0.D0 +gw_lndscl_sgh = .true. +gw_oro_south_fac = 1.d0 +gw_prndl = 0.25d0 +gw_qbo_hdepth_scaling = 1.0D0 +gw_top_taper = .false. +pgwv = 0 +tau_0_ubc = .false. + +! output fields +hist_add_inst_fields;h1: T, Q, U, V, PS +hist_output_frequency;h1: 1*nsteps +hist_write_nstep0;h1: .true. +hist_precision;h1: REAL64 +hist_max_frames;h1: 1 diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam7_derecho/shell_commands b/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam7_derecho/shell_commands new file mode 100644 index 00000000..5c619d67 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam7_derecho/shell_commands @@ -0,0 +1 @@ + ./xmlchange CAM_CONFIG_OPTS="--dyn none --physics-suites gw_cam7_se" diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam7_derecho/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam7_derecho/user_nl_cam new file mode 100644 index 00000000..eb3f83e5 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam7_derecho/user_nl_cam @@ -0,0 +1,51 @@ +! fhistc_ltso snapshot +pver = 58 +ncdata = '/glade/campaign/cesm/community/amwg/sima_baselines/cam_sima_test_snapshots/cam_ne3pg3_fhistc_ltso_gw_drag_cam7se_snapshot_derecho_gnu_before_c20251013.nc' +ncdata_check = '/glade/campaign/cesm/community/amwg/sima_baselines/cam_sima_test_snapshots/cam_ne3pg3_fhistc_ltso_gw_drag_cam7se_snapshot_derecho_gnu_after_c20251013.nc' + +debug_output = 0 + +! tolerances for testing +ncdata_check_err = .true. +min_difference = 2e-15 + +! topo file for beta ridge scheme input +bnd_topo = '/glade/campaign/cesm/cesmdata/inputdata/atm/cam/topo/se/ne3pg3_gmted2010_modis_bedmachine_nc0540_Laplace1000_noleak_20230209.nc' + +gw_movmtn_use_pbl_source = .true. ! se dycore +gw_movmtn_alpha = 0.01d0 +effgw_beres_dp = 0.15D0 +effgw_cm = 1.D0 +effgw_movmtn_pbl = 1.0d0 +effgw_rdg_beta = 0.5D0 +effgw_rdg_beta_max = 0.5D0 +front_gaussian_width = 30.D0 +gw_frontgfc = 1.25D-15 +gw_apply_tndmax = .false. +gw_dc = 2.5D0 +gw_dc_long = 0.D0 +gw_drag_file_dp = '/glade/campaign/cesm/cesmdata/inputdata/atm/waccm/gw/newmfspectra40_dc25.nc' +gw_drag_file_movmtn = '/glade/campaign/cesm/cesmdata/inputdata/atm/waccm/gw/mfc0lookup_mm.nc' +gw_lndscl_sgh = .true. +gw_oro_south_fac = 1.d0 +gw_polar_taper = .false. +gw_prndl = 0.5D0 +gw_qbo_hdepth_scaling = 0.32D0 +gw_movmtn_plaunch = 32500.0d0 +gw_movmtn_psteer = 65000.0d0 +gw_movmtn_source = 1 +gw_rdg_beta_nrdg = 10 +pgwv = 32 +pgwv_long = 0 +gw_rdg_beta_cd_llb = 1.0D0 +tau_0_ubc = .true. +gw_front_taubgnd = 1.5D-3 +gw_rdg_beta_trpd_leewv = .false. + +! output fields +hist_add_inst_fields;h1: T, Q, U, V, PS +hist_output_frequency;h1: 1*nsteps +hist_write_nstep0;h1: .true. +hist_precision;h1: REAL64 +hist_max_frames;h1: 1 + From edf0bd764b75a18f511561e6d624bafc3f09c2b2 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Tue, 14 Oct 2025 14:03:05 -0400 Subject: [PATCH 101/150] Remove UVTPS outfields from snapshot gw tests as sima_state_diagnostics does not run here --- .../testmods_dirs/cam/outfrq_gw_cam4_derecho/user_nl_cam | 7 ------- .../testmods_dirs/cam/outfrq_gw_cam7_derecho/user_nl_cam | 8 -------- 2 files changed, 15 deletions(-) diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam4_derecho/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam4_derecho/user_nl_cam index def1e644..4f071621 100644 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam4_derecho/user_nl_cam +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam4_derecho/user_nl_cam @@ -19,10 +19,3 @@ gw_qbo_hdepth_scaling = 1.0D0 gw_top_taper = .false. pgwv = 0 tau_0_ubc = .false. - -! output fields -hist_add_inst_fields;h1: T, Q, U, V, PS -hist_output_frequency;h1: 1*nsteps -hist_write_nstep0;h1: .true. -hist_precision;h1: REAL64 -hist_max_frames;h1: 1 diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam7_derecho/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam7_derecho/user_nl_cam index eb3f83e5..6243140e 100644 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam7_derecho/user_nl_cam +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_gw_cam7_derecho/user_nl_cam @@ -41,11 +41,3 @@ gw_rdg_beta_cd_llb = 1.0D0 tau_0_ubc = .true. gw_front_taubgnd = 1.5D-3 gw_rdg_beta_trpd_leewv = .false. - -! output fields -hist_add_inst_fields;h1: T, Q, U, V, PS -hist_output_frequency;h1: 1*nsteps -hist_write_nstep0;h1: .true. -hist_precision;h1: REAL64 -hist_max_frames;h1: 1 - From 0b21825cd931652143909b443e81f4d7c2aaca77 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Tue, 14 Oct 2025 20:12:02 -0400 Subject: [PATCH 102/150] Provide frontgf, frontga from dycore (SE); remove from cam_runtime_opts gw flags; set initial_value for ttend_dp so it persist across bc/ac --- src/control/runtime_obj.F90 | 26 ++----------------- src/control/runtime_opts.F90 | 10 +------- src/data/registry.xml | 1 + src/dynamics/se/dp_coupling.F90 | 44 ++++++++++----------------------- src/dynamics/se/dyn_comp.F90 | 2 ++ 5 files changed, 19 insertions(+), 64 deletions(-) diff --git a/src/control/runtime_obj.F90 b/src/control/runtime_obj.F90 index 157545a8..30d4acce 100644 --- a/src/control/runtime_obj.F90 +++ b/src/control/runtime_obj.F90 @@ -20,10 +20,6 @@ module runtime_obj type, public :: runtime_options character(len=CS), private :: phys_suite = unset_str character(len=16), private :: waccmx_opt = unset_str - ! use_gw_front: Frontogenesis - logical, private :: use_gw_front = .false. - ! use_gw_front_igw: Frontogenesis to inertial spectrum. - logical, private :: use_gw_front_igw = .false. ! update_thermo_variables: update thermo "constants" to composition-dependent thermo variables logical, private :: update_thermo_variables = .false. contains @@ -74,20 +70,6 @@ pure character(len=16) function waccmx_option(self) end function waccmx_option - pure logical function gw_front(self) - class(runtime_options), intent(in) :: self - - gw_front = self%use_gw_front - - end function gw_front - - pure logical function gw_front_igw(self) - class(runtime_options), intent(in) :: self - - gw_front_igw = self%use_gw_front_igw - - end function gw_front_igw - pure logical function update_thermodynamic_variables(self) class(runtime_options), intent(in) :: self @@ -95,14 +77,12 @@ pure logical function update_thermodynamic_variables(self) end function update_thermodynamic_variables - subroutine cam_set_runtime_opts(phys_suite, waccmx_opt, & - gw_front, gw_front_igw) + subroutine cam_set_runtime_opts(phys_suite, waccmx_opt) use cam_abortutils, only: endrun + ! Initialize the CAM runtime object character(len=CS), intent(in) :: phys_suite character(len=16), intent(in) :: waccmx_opt - logical, intent(in) :: gw_front - logical, intent(in) :: gw_front_igw if (runtime_configured) then ! We might need more action to reset this so do not allow it now @@ -111,8 +91,6 @@ subroutine cam_set_runtime_opts(phys_suite, waccmx_opt, & cam_runtime_opts%phys_suite = trim(phys_suite) cam_runtime_opts%waccmx_opt = trim(waccmx_opt) - cam_runtime_opts%use_gw_front = gw_front - cam_runtime_opts%use_gw_front_igw = gw_front_igw cam_runtime_opts%update_thermo_variables = (trim(waccmx_opt) == 'ionosphere' .or. & trim(waccmx_opt) == 'neutral') diff --git a/src/control/runtime_opts.F90 b/src/control/runtime_opts.F90 index 0951ca1f..2a88f175 100644 --- a/src/control/runtime_opts.F90 +++ b/src/control/runtime_opts.F90 @@ -65,15 +65,11 @@ subroutine read_namelist(nlfilename, single_column, scmlat, scmlon) character(len=512) :: errmsg character(len=64), allocatable :: schemes(:) integer :: errflg - logical :: use_gw_front - logical :: use_gw_front_igw character(len=*), parameter :: subname = "read_namelist" ! Initialize system-wide runtime configuration variables waccmx_opt = unset_str - use_gw_front = .false. - use_gw_front_igw = .false. !----------------------------------------------------------------------- ! Call subroutines for modules to read their own namelist. @@ -118,12 +114,8 @@ subroutine read_namelist(nlfilename, single_column, scmlat, scmlon) call cam_read_ccpp_scheme_namelists(nlfilename, schemes, & mpicom, masterprocid, masterproc, iulog) - ! XXgoldyXX: Need to figure out how to grab gravity-wave (and other) - ! scheme variables if and only if they are compiled in. - ! Finally, set the system-wide runtime configuration object - call cam_set_runtime_opts(phys_suite_name, waccmx_opt, & - use_gw_front, use_gw_front_igw) + call cam_set_runtime_opts(phys_suite_name, waccmx_opt) end subroutine read_namelist diff --git a/src/data/registry.xml b/src/data/registry.xml index d279e427..01b7bbb4 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -1558,6 +1558,7 @@ units="K s-1" type="real" kind="kind_phys" allocatable="allocatable"> horizontal_dimension vertical_layer_dimension + 0.0_kind_phys pbuf_TTEND_DP diff --git a/src/dynamics/se/dp_coupling.F90 b/src/dynamics/se/dp_coupling.F90 index 572f663f..b60c4a4a 100644 --- a/src/dynamics/se/dp_coupling.F90 +++ b/src/dynamics/se/dp_coupling.F90 @@ -158,28 +158,19 @@ subroutine d_p_coupling(cam_runtime_opts, phys_state, phys_tend, dyn_out) call check_allocate(ierr, subname, 'omega_tmp(nphys_pts,pver,nelemd)', & file=__FILE__, line=__LINE__) - if (cam_runtime_opts%gw_front() .or. & - cam_runtime_opts%gw_front_igw()) then - - allocate(frontgf(nphys_pts,pver,nelemd), stat=ierr) - call check_allocate(ierr, subname, 'frontgf(nphys_pts,pver,nelemd)', & - file=__FILE__, line=__LINE__) + allocate(frontgf(nphys_pts,pver,nelemd), stat=ierr) + call check_allocate(ierr, subname, 'frontgf(nphys_pts,pver,nelemd)', & + file=__FILE__, line=__LINE__) - allocate(frontga(nphys_pts,pver,nelemd), stat=ierr) - call check_allocate(ierr, subname, 'frontga(nphys_pts,pver,nelemd)', & - file=__FILE__, line=__LINE__) - end if + allocate(frontga(nphys_pts,pver,nelemd), stat=ierr) + call check_allocate(ierr, subname, 'frontga(nphys_pts,pver,nelemd)', & + file=__FILE__, line=__LINE__) if (iam < par%nprocs) then - ! Gravity Waves - if (cam_runtime_opts%gw_front() .or. & - cam_runtime_opts%gw_front_igw()) then - - ! Calculate frontogenesis function and angle - call gws_src_fnct(elem, tl_f, tl_qdp_np0, frontgf, frontga, nphys) - - end if + ! Calculate frontogenesis function and angle + ! for gravity wave parameterization. + call gws_src_fnct(elem, tl_f, tl_qdp_np0, frontgf, frontga, nphys) if (fv_nphys > 0) then call test_mapping_overwrite_dyn_state(elem,dyn_out%fvm) @@ -240,14 +231,8 @@ subroutine d_p_coupling(cam_runtime_opts, phys_state, phys_tend, dyn_out) omega_tmp(:,:,:) = 0._r8 phis_tmp(:,:) = 0._r8 q_tmp(:,:,:,:) = 0._r8 - - if (cam_runtime_opts%gw_front() .or. & - cam_runtime_opts%gw_front_igw()) then - - frontgf(:,:,:) = 0._r8 - frontga(:,:,:) = 0._r8 - - end if + frontgf(:,:,:) = 0._r8 + frontga(:,:,:) = 0._r8 endif ! iam < par%nprocs @@ -275,11 +260,8 @@ subroutine d_p_coupling(cam_runtime_opts, phys_state, phys_tend, dyn_out) phys_state%u(icol, ilyr) = real(uv_tmp(blk_ind(1), 1, ilyr, ie), kind_phys) phys_state%v(icol, ilyr) = real(uv_tmp(blk_ind(1), 2, ilyr, ie), kind_phys) phys_state%omega(icol, ilyr) = real(omega_tmp(blk_ind(1), ilyr, ie), kind_phys) - - if (cam_runtime_opts%gw_front() .or. cam_runtime_opts%gw_front_igw()) then - phys_state%frontgf(icol, ilyr) = real(frontgf(blk_ind(1), ilyr, ie), kind_phys) - phys_state%frontga(icol, ilyr) = real(frontga(blk_ind(1), ilyr, ie), kind_phys) - end if + phys_state%frontgf(icol, ilyr) = real(frontgf(blk_ind(1), ilyr, ie), kind_phys) + phys_state%frontga(icol, ilyr) = real(frontga(blk_ind(1), ilyr, ie), kind_phys) end do do m = 1, num_advected diff --git a/src/dynamics/se/dyn_comp.F90 b/src/dynamics/se/dyn_comp.F90 index e3ddeb7d..aea91d37 100644 --- a/src/dynamics/se/dyn_comp.F90 +++ b/src/dynamics/se/dyn_comp.F90 @@ -1864,6 +1864,8 @@ subroutine read_inidat(dyn_in) call mark_as_initialized("tendency_of_eastward_wind_due_to_model_physics") call mark_as_initialized("tendency_of_northward_wind_due_to_model_physics") call mark_as_initialized("specific_heat_of_air_used_in_dycore") + call mark_as_initialized("frontogenesis_function") + call mark_as_initialized("frontogenesis_angle") ! These energy variables are calculated by check_energy_timestep_init ! but need to be marked here From f3aa434f9d39255784428812cab2410a349c84fa Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Tue, 14 Oct 2025 20:17:22 -0400 Subject: [PATCH 103/150] Remove remnant gw_front(_igw) from runtime obj. --- src/control/runtime_obj.F90 | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/control/runtime_obj.F90 b/src/control/runtime_obj.F90 index 30d4acce..ff27e163 100644 --- a/src/control/runtime_obj.F90 +++ b/src/control/runtime_obj.F90 @@ -29,8 +29,6 @@ module runtime_obj ! Runtime parameters of interest to dycore procedure, public :: waccmx_on procedure, public :: waccmx_option - procedure, public :: gw_front - procedure, public :: gw_front_igw procedure, public :: update_thermodynamic_variables end type runtime_options From 4594f2c3e936bbaac67a95a27e6c7df6e88f70f2 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Tue, 14 Oct 2025 20:40:32 -0400 Subject: [PATCH 104/150] Add bnd_topo to FCAM7 testmods; port vorticity code from CAM to SE dycore in SIMA --- .../cam/outfrq_se_cslam_analy_ic/user_nl_cam | 8 ++ src/dynamics/se/dp_coupling.F90 | 14 ++ src/dynamics/se/dyn_comp.F90 | 3 +- src/dynamics/se/gravity_waves_sources.F90 | 127 ++++++++++++++++++ 4 files changed, 151 insertions(+), 1 deletion(-) diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_se_cslam_analy_ic/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq_se_cslam_analy_ic/user_nl_cam index ddd9f580..69eb4b4f 100644 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq_se_cslam_analy_ic/user_nl_cam +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_se_cslam_analy_ic/user_nl_cam @@ -2,3 +2,11 @@ hist_add_inst_fields;h1: T, Q, U, V, PS hist_output_frequency;h1: 1*nsteps hist_write_nstep0;h1: .true. analytic_ic_type=held_suarez_1994 + +! generally, analytic_ic runs do not need a topo file (see namelist_definition_cam.xml) +! however, a topo file needs to be supplied to run the Beta Ridge gravity wave scheme. +! +! in the future, the FCAM7 test can be migrated to using 'real' initial conditions, +! and this line can be removed, and +! analytic ic runs can go back to having no topo file supplied as it is by default. +bnd_topo='/glade/campaign/cesm/cesmdata/inputdata/atm/cam/topo/se/ne3pg3_gmted2010_modis_bedmachine_nc0540_Laplace1000_noleak_20230209.nc' diff --git a/src/dynamics/se/dp_coupling.F90 b/src/dynamics/se/dp_coupling.F90 index b60c4a4a..5eb9f981 100644 --- a/src/dynamics/se/dp_coupling.F90 +++ b/src/dynamics/se/dp_coupling.F90 @@ -53,6 +53,7 @@ subroutine d_p_coupling(cam_runtime_opts, phys_state, phys_tend, dyn_out) ! dry air mass. use gravity_waves_sources, only: gws_src_fnct + use gravity_waves_sources, only: gws_src_vort use hycoef, only: hyai, ps0 use test_fvm_mapping, only: test_mapping_overwrite_dyn_state, test_mapping_output_phys_state use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t @@ -92,6 +93,9 @@ subroutine d_p_coupling(cam_runtime_opts, phys_state, phys_tend, dyn_out) real (kind=r8), allocatable :: frontgf(:,:,:) ! temp arrays to hold frontogenesis real (kind=r8), allocatable :: frontga(:,:,:) ! function (frontgf) and angle (frontga) + ! Vorticity + real (kind=r8), allocatable :: vort4gw(:,:,:) ! temp arrays to hold vorticity + integer :: ncols,ierr integer :: blk_ind(1), m, m_cnst integer :: nphys @@ -166,12 +170,20 @@ subroutine d_p_coupling(cam_runtime_opts, phys_state, phys_tend, dyn_out) call check_allocate(ierr, subname, 'frontga(nphys_pts,pver,nelemd)', & file=__FILE__, line=__LINE__) + allocate(vort4gw(nphys_pts,pver,nelemd), stat=ierr) + call check_allocate(ierr, subname, 'vort4gw(nphys_pts,pver,nelemd)', & + file=__FILE__, line=__LINE__) + + if (iam < par%nprocs) then ! Calculate frontogenesis function and angle ! for gravity wave parameterization. call gws_src_fnct(elem, tl_f, tl_qdp_np0, frontgf, frontga, nphys) + ! Calculate vorticity for moving mountain gravity wave parameterization. + call gws_src_vort(elem, tl_f, tl_qdp_np0, vort4gw, nphys) + if (fv_nphys > 0) then call test_mapping_overwrite_dyn_state(elem,dyn_out%fvm) !****************************************************************** @@ -233,6 +245,7 @@ subroutine d_p_coupling(cam_runtime_opts, phys_state, phys_tend, dyn_out) q_tmp(:,:,:,:) = 0._r8 frontgf(:,:,:) = 0._r8 frontga(:,:,:) = 0._r8 + vort4gw(:,:,:) = 0._r8 endif ! iam < par%nprocs @@ -262,6 +275,7 @@ subroutine d_p_coupling(cam_runtime_opts, phys_state, phys_tend, dyn_out) phys_state%omega(icol, ilyr) = real(omega_tmp(blk_ind(1), ilyr, ie), kind_phys) phys_state%frontgf(icol, ilyr) = real(frontgf(blk_ind(1), ilyr, ie), kind_phys) phys_state%frontga(icol, ilyr) = real(frontga(blk_ind(1), ilyr, ie), kind_phys) + phys_state%vorticity(icol, ilyr) = real(vort4gw(blk_ind(1), ilyr, ie), kind_phys) end do do m = 1, num_advected diff --git a/src/dynamics/se/dyn_comp.F90 b/src/dynamics/se/dyn_comp.F90 index aea91d37..277d875b 100644 --- a/src/dynamics/se/dyn_comp.F90 +++ b/src/dynamics/se/dyn_comp.F90 @@ -838,7 +838,8 @@ subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) call prim_init2(elem, fvm, hybrid, nets, nete, TimeLevel, hvcoord) !$OMP END PARALLEL - if (cam_runtime_opts%gw_front() .or. cam_runtime_opts%gw_front_igw()) call gws_init(elem) + ! initialize gravity wave sources + call gws_init(elem) end if ! iam < par%nprocs !Remove/replace after CAMDEN history output is enabled -JN: diff --git a/src/dynamics/se/gravity_waves_sources.F90 b/src/dynamics/se/gravity_waves_sources.F90 index b3484c7c..61c4dd4b 100644 --- a/src/dynamics/se/gravity_waves_sources.F90 +++ b/src/dynamics/se/gravity_waves_sources.F90 @@ -18,8 +18,10 @@ module gravity_waves_sources !! for use by WACCM (via dp_coupling) public :: gws_src_fnct + public :: gws_src_vort public :: gws_init private :: compute_frontogenesis + private :: compute_vorticity_4gw type (EdgeBuffer_t) :: edge3 type (derivative_t) :: deriv @@ -111,6 +113,131 @@ subroutine gws_src_fnct(elem, tl, tlq, frontgf, frontga,nphys) end subroutine gws_src_fnct +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + subroutine gws_src_vort(elem, tl, tlq, vort4gw, nphys) + use derivative_mod, only : derivinit + use dimensions_mod, only : nelemd + use dof_mod, only : UniquePoints + use hybrid_mod, only : config_thread_region, get_loop_ranges + use parallel_mod, only : par + use ppgrid, only : pver + use thread_mod, only : horz_num_threads + use dimensions_mod, only : fv_nphys + use cam_abortutils, only : handle_allocate_error + + implicit none + type (element_t), intent(in), dimension(:) :: elem + integer, intent(in) :: tl, nphys, tlq + + ! + real (kind=r8), intent(out) :: vort4gw(nphys*nphys,pver,nelemd) + + ! Local variables + type (hybrid_t) :: hybrid + integer :: nets, nete, ithr, ncols, ie, ierr + + ! + real(kind=r8), allocatable :: vort4gw_thr(:,:,:,:) + + ! This does not need to be a thread private data-structure + call derivinit(deriv) + !!$OMP PARALLEL NUM_THREADS(horz_num_threads), DEFAULT(SHARED), PRIVATE(nets,nete,hybrid,ie,ncols,vort4gw_thr) + hybrid = config_thread_region(par,'serial') + call get_loop_ranges(hybrid,ibeg=nets,iend=nete) + + allocate(vort4gw_thr(nphys,nphys,nlev,nets:nete), stat=ierr) + call handle_allocate_error(ierr, 'gws_src_vort', 'vort4gw_thr') + + call compute_vorticity_4gw(vort4gw_thr,tl,tlq,elem,deriv,hybrid,nets,nete,nphys) + + if (fv_nphys>0) then + do ie=nets,nete + vort4gw(:,:,ie) = RESHAPE(vort4gw_thr(:,:,:,ie),(/nphys*nphys,nlev/)) + end do + else + do ie=nets,nete + ncols = elem(ie)%idxP%NumUniquePts + call UniquePoints(elem(ie)%idxP, nlev, vort4gw_thr(:,:,:,ie), vort4gw(1:ncols,:,ie)) + end do + end if + deallocate(vort4gw_thr) + + !!$OMP END PARALLEL + + end subroutine gws_src_vort + + subroutine compute_vorticity_4gw(vort4gw,tl,tlq,elem,ederiv,hybrid,nets,nete,nphys) + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! compute vorticity for use in gw params + ! F = ( curl ) [U,V] + ! + ! Original by Peter Lauritzen, Julio Bacmeister*, Dec 2024 + ! Patterned on 'compute_frontogenesis' + ! + ! * corresponding/blame-able + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + use derivative_mod, only: vorticity_sphere + use edge_mod, only: edgevpack, edgevunpack + use bndry_mod, only: bndry_exchange + use dimensions_mod, only: fv_nphys + use fvm_mapping, only: dyn2phys + + type(hybrid_t), intent(in) :: hybrid + type(element_t), intent(in) :: elem(:) + type(derivative_t), intent(in) :: ederiv + integer, intent(in) :: nets,nete,nphys + integer, intent(in) :: tl,tlq + real(r8), intent(out) :: vort4gw(nphys,nphys,nlev,nets:nete) + + ! local + real(r8) :: area_inv(fv_nphys,fv_nphys), tmp(np,np) + real(r8) :: vort_gll(np,np,nlev,nets:nete) + integer :: k,kptr,i,j,ie,component,h,nq,m_cnst,n0 + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! First calculate vorticity on GLL grid + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! set timelevel=1 fro velocities + n0=tl + do ie=nets,nete + do k=1,nlev + call vorticity_sphere(elem(ie)%state%v(:,:,:,k,n0),ederiv,elem(ie),vort_gll(:,:,k,ie)) + end do + do k=1,nlev + vort_gll(:,:,k,ie) = vort_gll(:,:,k,ie)*elem(ie)%spheremp(:,:) + end do + ! pack + call edgeVpack(edge1, vort_gll(:,:,:,ie),nlev,0,ie) + enddo + call bndry_exchange(hybrid,edge1,location='compute_vorticity_4gw') + do ie=nets,nete + call edgeVunpack(edge1, vort_gll(:,:,:,ie),nlev,0,ie) + ! apply inverse mass matrix, + do k=1,nlev + vort_gll(:,:,k,ie) = vort_gll(:,:,k,ie)*elem(ie)%rspheremp(:,:) + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! Now regrid from GLL to PhysGrid if necessary + ! otherwise just return vorticity on GLL grid + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + if (fv_nphys>0) then + tmp = 1.0_r8 + area_inv = dyn2phys(tmp,elem(ie)%metdet) + area_inv = 1.0_r8/area_inv + do k=1,nlev + vort4gw(:,:,k,ie) = dyn2phys( vort_gll(:,:,k,ie) , elem(ie)%metdet , area_inv ) + end do + else + do k=1,nlev + vort4gw(:,:,k,ie) = vort_gll(:,:,k,ie) + end do + end if + enddo + + + end subroutine compute_vorticity_4gw + subroutine compute_frontogenesis(frontgf,frontga,tl,tlq,elem,ederiv,hybrid,nets,nete,nphys) !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! compute frontogenesis function F From 241994b2bee8b1622007a51f4cbf4d6d9e0da947 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Tue, 14 Oct 2025 20:44:05 -0400 Subject: [PATCH 105/150] Fix port of vorticity code --- src/dynamics/se/gravity_waves_sources.F90 | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/dynamics/se/gravity_waves_sources.F90 b/src/dynamics/se/gravity_waves_sources.F90 index 61c4dd4b..55e0f0b5 100644 --- a/src/dynamics/se/gravity_waves_sources.F90 +++ b/src/dynamics/se/gravity_waves_sources.F90 @@ -23,7 +23,7 @@ module gravity_waves_sources private :: compute_frontogenesis private :: compute_vorticity_4gw - type (EdgeBuffer_t) :: edge3 + type (EdgeBuffer_t) :: edge3,edge1 type (derivative_t) :: deriv real(r8) :: psurf_ref @@ -45,6 +45,7 @@ subroutine gws_init(elem) ! Set up variables similar to dyn_comp and prim_driver_mod initializations call initEdgeBuffer(par, edge3, elem, 3*nlev,nthreads=1) + call initEdgeBuffer(par, edge1, elem, nlev,nthreads=1) psurf_ref = hypi(nlevp) @@ -123,7 +124,7 @@ subroutine gws_src_vort(elem, tl, tlq, vort4gw, nphys) use ppgrid, only : pver use thread_mod, only : horz_num_threads use dimensions_mod, only : fv_nphys - use cam_abortutils, only : handle_allocate_error + use cam_abortutils, only : check_allocate implicit none type (element_t), intent(in), dimension(:) :: elem @@ -139,6 +140,8 @@ subroutine gws_src_vort(elem, tl, tlq, vort4gw, nphys) ! real(kind=r8), allocatable :: vort4gw_thr(:,:,:,:) + character(len=*), parameter :: subname = 'gws_src_vort' + ! This does not need to be a thread private data-structure call derivinit(deriv) !!$OMP PARALLEL NUM_THREADS(horz_num_threads), DEFAULT(SHARED), PRIVATE(nets,nete,hybrid,ie,ncols,vort4gw_thr) @@ -146,7 +149,9 @@ subroutine gws_src_vort(elem, tl, tlq, vort4gw, nphys) call get_loop_ranges(hybrid,ibeg=nets,iend=nete) allocate(vort4gw_thr(nphys,nphys,nlev,nets:nete), stat=ierr) - call handle_allocate_error(ierr, 'gws_src_vort', 'vort4gw_thr') + call check_allocate(ierr, subname, & + 'vort4gw_thr(nphys,nphys,nlev,nets:nete)', & + file=__FILE__, line=__LINE__) call compute_vorticity_4gw(vort4gw_thr,tl,tlq,elem,deriv,hybrid,nets,nete,nphys) From 931067843c6272faa2e3191275623edb173e37f9 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Tue, 14 Oct 2025 20:46:11 -0400 Subject: [PATCH 106/150] Move vorticity (vort4gw) to physics_state --- src/data/registry.xml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/data/registry.xml b/src/data/registry.xml index 01b7bbb4..f505065e 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -138,6 +138,13 @@ horizontal_dimension vertical_layer_dimension frontga pbuf_FRONTGA + + horizontal_dimension vertical_layer_dimension + pbuf_VORT4GW + reciprocal_of_dimensionless_exner_function_wrt_surface_air_pressure frontogenesis_function frontogenesis_angle + relative_vorticity vertically_integrated_total_energy_using_dycore_energy_formula_at_start_of_physics_timestep vertically_integrated_total_energy_using_dycore_energy_formula vertically_integrated_total_water_at_start_of_physics_timestep @@ -1369,13 +1377,6 @@ - - horizontal_dimension vertical_layer_dimension - pbuf_VORT4GW - Date: Tue, 14 Oct 2025 20:58:18 -0400 Subject: [PATCH 107/150] Initialize vorticity in se dycore; registry initial_value for clubb fields --- src/data/registry.xml | 6 +++++- src/dynamics/se/dyn_comp.F90 | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/data/registry.xml b/src/data/registry.xml index f505065e..6aae0893 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -1353,12 +1353,14 @@ 273.15_kind_phys tpert pbuf_tpert + horizontal_dimension vertical_layer_dimension + 0.0_kind_phys pbuf_TTEND_CLUBB horizontal_dimension vertical_interface_dimension + 0.0_kind_phys pbuf_UPWP_CLUBB_GW horizontal_dimension vertical_interface_dimension + 0.0_kind_phys pbuf_VPWP_CLUBB_GW @@ -1407,7 +1411,7 @@ standard_name="flag_for_cloud_area_fraction_to_use_shallow_convection_calculated_cloud_area_fraction" units="flag" type="logical"> should cloud fraction use shallow convection calculated convective cloud area fraction - + .false. Date: Thu, 16 Oct 2025 14:27:01 -0400 Subject: [PATCH 108/150] Port cam6_4_126: Fix frontogenesis bug (SE dycore only) from CAM --- src/dynamics/se/gravity_waves_sources.F90 | 122 ++++++++++++++++++++-- 1 file changed, 111 insertions(+), 11 deletions(-) diff --git a/src/dynamics/se/gravity_waves_sources.F90 b/src/dynamics/se/gravity_waves_sources.F90 index 55e0f0b5..27ef3a6a 100644 --- a/src/dynamics/se/gravity_waves_sources.F90 +++ b/src/dynamics/se/gravity_waves_sources.F90 @@ -257,6 +257,8 @@ subroutine compute_frontogenesis(frontgf,frontga,tl,tlq,elem,ederiv,hybrid,nets, ! Integrated into gravity_waves_sources module, several arguments made global ! to prevent repeated allocation/initialization ! + ! Frontogenesis function correction by Walter Hannah, Mark Taylor, and Jack Chen. October 2025 + ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! use physconst, only: cappa use air_composition, only: dry_air_species_num,thermodynamic_active_species_num @@ -285,14 +287,45 @@ subroutine compute_frontogenesis(frontgf,frontga,tl,tlq,elem,ederiv,hybrid,nets, real(r8) :: frontga_gll(np,np,nlev,nets:nete) integer :: k,kptr,i,j,ie,component,h,nq,m_cnst real(r8) :: gradth(np,np,2,nlev,nets:nete) ! grad(theta) - real(r8) :: p(np,np) ! pressure at mid points - real(r8) :: pint(np,np) ! pressure at interface points - real(r8) :: theta(np,np) ! potential temperature at mid points + real(r8) :: p(np,np,nlev) ! pressure at mid points + real(r8) :: pint(np,np,nlev+1) ! pressure at interface points + real(r8) :: gradp(np,np,2) ! grad(pressure) + real(r8) :: theta(np,np,nlev) ! potential temperature at mid points + real(r8) :: dtheta_dp(np,np,nlev) ! d(theta)/dp for eta to pressure surface correction + real(r8) :: dum_grad(np,np,2) ! horizontal gradient of zonal and meridional wind on cartesian coordinate on isobaric surface + real(r8) :: dum_cart(np,np,3,nlev) ! zonal & meridional wind on cartesian coordinate + real(r8) :: ddp_dum_cart(np,np,3,nlev) ! vertical gradient of zonal & meridional wind on cartesian coordinate real(r8) :: C(np,np,2), sum_water(np,np) + ! By Mark Taylor + ! For a vector velocity "v", a tensor "grad(v)", and a vector "grad(theta)", + ! this loop computes the vector "grad(theta)*grad(v)" + ! + ! Representing the tensor "grad(v)" in spherical coordinates is difficult. This routine + ! avoids this by computing a mathematically equivalent form using a mixture of + ! Cartesian and spherical coordinates + ! + ! This routine is a modified version of derivative_mod.F90:ugradv_sphere() in that the + ! grad(v) term is modified to compute grad_p(v) - the gradient on p-surfaces expressed + ! in terms of the gradient on model surfaces and a vertical pressure gradient. + ! + ! First, v is represented in cartesian coordinates v(c) for c=1,2,3 + ! For each v(c), we compute its gradient on p-surfaces via: + ! grad(v(c)) - d(v(c))/dz grad(p) + ! Each of these gradients is represented in *spherical* coordinates (i=1,2) + ! + ! We then dot each of these vectors with grad(theta). This dot product is computed + ! in spherical coordinates. The end result is dum_cart(c), for c=1,2,3 + ! These three scalars are the three Cartesian coefficients of + ! the vector "grad(theta)*grad(v)" + ! + ! This Cartesian vector is then transformed back to spherical coordinates + ! + do ie=nets,nete ! pressure at model top - pint(:,:) = hvcoord%hyai(1) + pint(:,:,1) = hvcoord%hyai(1)*hvcoord%ps0 + do k=1,nlev ! moist pressure at mid points sum_water(:,:) = 1.0_r8 @@ -303,15 +336,49 @@ subroutine compute_frontogenesis(frontgf,frontga,tl,tlq,elem,ederiv,hybrid,nets, ! sum_water(:,:) = sum_water(:,:) + elem(ie)%state%Qdp(:,:,k,m_cnst,tlq)/elem(ie)%state%dp3d(:,:,k,tl) end do - p(:,:) = pint(:,:) + 0.5_r8*sum_water(:,:)*elem(ie)%state%dp3d(:,:,k,tl) + p(:,:,k) = pint(:,:,k) + 0.5_r8*sum_water(:,:)*elem(ie)%state%dp3d(:,:,k,tl) ! moist pressure at interface for next iteration - pint(:,:) = pint(:,:)+elem(ie)%state%dp3d(:,:,k,tl) + pint(:,:,k+1) = pint(:,:,k)+elem(ie)%state%dp3d(:,:,k,tl) ! - theta(:,:) = elem(ie)%state%T(:,:,k,tl)*(psurf_ref / p(:,:))**cappa - ! gradth(:,:,:,k,ie) = gradient_sphere(theta,ederiv,elem(ie)%Dinv) - call gradient_sphere(theta,ederiv,elem(ie)%Dinv,gradth(:,:,:,k,ie)) - ! compute C = (grad(theta) dot grad ) u - C(:,:,:) = ugradv_sphere(gradth(:,:,:,k,ie), elem(ie)%state%v(:,:,:,k,tl),ederiv,elem(ie)) + theta(:,:) = elem(ie)%state%T(:,:,k,tl)*(psurf_ref / p(:,:,k))**cappa + end do + + call compute_vertical_derivative(pint,p,theta,dtheta_dp) + + do k=1,nlev + call gradient_sphere(theta(:,:,k),ederiv,elem(ie)%Dinv,gradth(:,:,:,k,ie)) + + call gradient_sphere(p(:,:,k),ederiv,elem(ie)%Dinv,gradp) + + do component=1,2 + gradth(:,:,component,k,ie) = gradth(:,:,component,k,ie) - dtheta_dp(:,:,k) * gradp(:,:,component) + end do + end do + + do k=1,nlev + do component=1,3 + dum_cart(:,:,component,k) = sum( elem(ie)%vec_sphere2cart(:,:,component,:) * elem(ie)%state%v(:,:,:,k,tl),3 ) + end do + end do + + do component=1,3 + call compute_vertical_derivative(pint,p,dum_cart(:,:,component,:),ddp_dum_cart(:,:,component,:)) + end do + do k=1,nlev + call gradient_sphere(p(:,:,k),ederiv,elem(ie)%Dinv,gradp) + + do component=1,3 + call gradient_sphere(dum_cart(:,:,component,k),ederiv,elem(ie)%Dinv,dum_grad) + do i=1,2 + dum_grad(:,:,i) = dum_grad(:,:,i) - ddp_dum_cart(:,:,component,k) * gradp(:,:,i) + end do + dum_cart(:,:,component,k) = sum( gradth(:,:,:,k,ie) * dum_grad , 3 ) + end do + + do i=1,2 + C(:,:,i) = sum(dum_cart(:,:,:,k)*elem(ie)%vec_sphere2cart(:,:,:,i), 3) + end do + ! gradth dot C frontgf_gll(:,:,k,ie) = -( C(:,:,1)*gradth(:,:,1,k,ie) + C(:,:,2)*gradth(:,:,2,k,ie) ) ! apply mass matrix @@ -363,5 +430,38 @@ subroutine compute_frontogenesis(frontgf,frontga,tl,tlq,elem,ederiv,hybrid,nets, enddo end subroutine compute_frontogenesis + subroutine compute_vertical_derivative(pint,pmid,data,ddata_dp) + !--------------------------------------------------------------------------- + real(r8), intent(in ) :: pint(np,np,nlev+1) + real(r8), intent(in ) :: pmid(np,np,nlev) + real(r8), intent(in ) :: data(np,np,nlev) + real(r8), intent(out) :: ddata_dp(np,np,nlev) + !--------------------------------------------------------------------------- + integer :: k + real(r8) :: pint_above(np,np) ! pressure interpolated to interface above the current k mid-point + real(r8) :: pint_below(np,np) ! pressure interpolated to interface below the current k mid-point + real(r8) :: dint_above(np,np) ! data interpolated to interface above the current k mid-point + real(r8) :: dint_below(np,np) ! data interpolated to interface below the current k mid-point + !--------------------------------------------------------------------------- + do k = 1,nlev + if (k==1) then + pint_above = pmid(:,:,k) + pint_below = pint(:,:,k+1) + dint_above = data(:,:,k) + dint_below = ( data(:,:,k+1) + data(:,:,k) ) / 2.0_r8 + elseif (k==nlev) then + pint_above = pint(:,:,k) + pint_below = pmid(:,:,k) + dint_above = ( data(:,:,k-1) + data(:,:,k) ) / 2.0_r8 + dint_below = data(:,:,k) + else + pint_above = pint(:,:,k) + pint_below = pint(:,:,k+1) + dint_above = ( data(:,:,k-1) + data(:,:,k) ) / 2.0_r8 + dint_below = ( data(:,:,k+1) + data(:,:,k) ) / 2.0_r8 + end if + ddata_dp(:,:,k) = ( dint_above - dint_below ) / ( pint_above - pint_below ) + end do + end subroutine compute_vertical_derivative end module gravity_waves_sources From c963e10dba0265d2c60c58bdac7f2d7348c8ce9e Mon Sep 17 00:00:00 2001 From: peverwhee Date: Mon, 20 Oct 2025 14:44:13 -0600 Subject: [PATCH 109/150] remove unused diagnostic field; add handling for parameters in registry --- .gitmodules | 4 +- ccpp_framework | 2 +- .../cam/outfrq_rrtmgp_derecho/shell_commands | 1 + .../cam/outfrq_rrtmgp_derecho/user_nl_cam | 3 +- src/control/cam_comp.F90 | 4 ++ src/data/cam_var_init_marks.inc | 43 +++++++++++++++++++ src/data/cam_var_init_marks_decl.inc | 1 + src/data/generate_registry_data.py | 2 +- src/data/registry.xml | 8 +++- src/physics/ncar_ccpp | 2 +- 10 files changed, 62 insertions(+), 8 deletions(-) diff --git a/.gitmodules b/.gitmodules index f9558978..c968e015 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "ccpp-framework"] path = ccpp_framework url = https://github.com/NCAR/ccpp-framework - fxtag = 2025-10-01-dev + fxtag = 2025-10-15-dev fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework [submodule "history"] @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = 16c146f + fxtag = b0cf14d631c641c69280f7b7ad81f61cc921af50 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/ccpp_framework b/ccpp_framework index d20a1858..37db3485 160000 --- a/ccpp_framework +++ b/ccpp_framework @@ -1 +1 @@ -Subproject commit d20a1858f9ce0d12c1cbe2757fcbdb979c421333 +Subproject commit 37db3485e1f71a2556df78c6636a96ed0ce137bc diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/shell_commands b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/shell_commands index 9d29c453..ce09f06f 100644 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/shell_commands +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/shell_commands @@ -1 +1,2 @@ ./xmlchange CAM_CONFIG_OPTS="--dyn none --physics-suites rrtmgp" +./xmlchange RUN_STARTDATE=1979-01-01 diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/user_nl_cam index e862a983..bfe753b9 100644 --- a/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/user_nl_cam +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho/user_nl_cam @@ -22,5 +22,4 @@ hist_add_inst_fields;h1: TOT_CLD_VISTAU,TOT_ICLD_VISTAU,ICE_ICLD_VISTAU,LIQ_ICLD hist_add_inst_fields;h1: QRL,QRLC,FLNT,FLNTC,FLUT,FLUTC,LWCF,FLN200,FLN200C,FLNR,FLNS,FLNSC,FLDS,FLDSC,FUL,FDL,FULC,FDLC ! Shortwave diagnostic fields hist_add_inst_fields;h1: SOLIN,QRS,QRSC,FSNT,FSNTC,FSNTOA,FSNTOAC,SWCF,FSUTOA,FSN200,FSN200C,FSNR,SOLL,SOLS,SOLLD,SOLSD -hist_add_inst_fields;h1: FSNS,FSNSC,FSDS,FSDSC,FSNIRTOA,FSNRTOAC,FSNRTOAS,FUS,FDS,FUSC,FDSC -hist_precision;h1: REAL64 +hist_add_inst_fields;h1: FSNS,FSNSC,FSDS,FSDSC,FUS,FDS,FUSC,FDSC diff --git a/src/control/cam_comp.F90 b/src/control/cam_comp.F90 index a41c107c..a10290be 100644 --- a/src/control/cam_comp.F90 +++ b/src/control/cam_comp.F90 @@ -30,6 +30,7 @@ module cam_comp use physics_types, only: dtime_phys use physics_types, only: calday, next_calday, radiation_offset, nextsw_cday use physics_types, only: is_first_timestep, nstep + use physics_types, only: is_first_restart_timestep use dyn_comp, only: dyn_import_t, dyn_export_t use perf_mod, only: t_barrierf, t_startf, t_stopf @@ -178,6 +179,9 @@ subroutine cam_init(caseid, ctitle, model_doi_url, & is_first_timestep = .true. call mark_as_initialized('is_first_timestep') + is_first_restart_timestep = .false. + call mark_as_initialized('is_first_restart_timestep') + nstep = get_nstep() call mark_as_initialized('current_timestep_number') diff --git a/src/data/cam_var_init_marks.inc b/src/data/cam_var_init_marks.inc index f1968b24..dd83c4bd 100644 --- a/src/data/cam_var_init_marks.inc +++ b/src/data/cam_var_init_marks.inc @@ -110,6 +110,49 @@ end function is_initialized + logical function is_parameter(varname, error_on_not_found) + + ! This function checks if the variable, , is + ! a parameter according to the 'initialized_vars' array. + + use cam_abortutils, only: endrun + + ! Dummy argument + character(len=*), intent(in) :: varname ! Variable name being checked + logical, optional, intent(in) :: error_on_not_found + + ! Local variables + integer :: stdnam_idx ! Standard name array index + logical :: found ! Check that was found + logical :: error_on_not_found_loc + character(len=*), parameter :: subname = 'is_parameter: ' + + is_parameter = .false. + found = .false. + + if (present(error_on_not_found)) then + error_on_not_found_loc = error_on_not_found + else + error_on_not_found_loc = .true. + end if + + ! Check if variable is a parameter + do stdnam_idx = 1, phys_var_num + if (trim(phys_var_stdnames(stdnam_idx)) == trim(varname)) then + is_parameter = (initialized_vars(stdnam_idx) == PARAM) + found = .true. + exit ! Exit loop once variable has been found and checked + end if + end do + + if (.not. found .and. error_on_not_found_loc) then + ! This condition is an internal error, it should not happen + call endrun(subname//": Variable '"//trim(varname)// & + "' is missing from phys_var_stdnames array.") + end if + + end function is_parameter + subroutine is_read_from_file(varname, is_read, stdnam_idx_out) ! This subroutine checks if the variable, , is read from diff --git a/src/data/cam_var_init_marks_decl.inc b/src/data/cam_var_init_marks_decl.inc index dbbe5a87..1e6ac19f 100644 --- a/src/data/cam_var_init_marks_decl.inc +++ b/src/data/cam_var_init_marks_decl.inc @@ -2,6 +2,7 @@ public :: mark_as_initialized public :: mark_as_read_from_file public :: is_initialized + public :: is_parameter public :: is_read_from_file !! Parameterized initialized_vars options - order matters diff --git a/src/data/generate_registry_data.py b/src/data/generate_registry_data.py index f858c60a..e2efdce6 100755 --- a/src/data/generate_registry_data.py +++ b/src/data/generate_registry_data.py @@ -306,7 +306,7 @@ def write_initial_value(self, outfile, indent, init_var, ddt_str, physconst_vars raise TypeError(emsg.format(var_name, self.var_type)) # end if elif init_val: - outfile.write(f"if ({init_var} .and. .not. is_initialized('{self.standard_name}', error_on_not_found=.false.)) then", indent) + outfile.write(f"if ({init_var} .and. (.not. is_initialized('{self.standard_name}', error_on_not_found=.false.) .or. is_parameter('{self.standard_name}', error_on_not_found=.false.))) then", indent) outfile.write(f"{var_name} = {init_val}", indent+1) if self.initial_val_vars and self.initial_val_vars.issubset(physconst_vars): outfile.write(f"call mark_as_initialized('{self.standard_name}')", indent+1) diff --git a/src/data/registry.xml b/src/data/registry.xml index f83e34ba..9f2580f7 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -36,6 +36,7 @@ + @@ -367,6 +368,11 @@ units="flag" type="logical"> flag indicating if it is the first timestep of an initial run + + flag indicating if it is the first timestep of restart run + horizontal_dimension vertical_layer_dimension number_of_ccpp_constituents - 1 + 1._kind_phys diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 16c146fe..b0cf14d6 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 16c146fe5167f1224a8cd88eb3b77ec9f8033291 +Subproject commit b0cf14d631c641c69280f7b7ad81f61cc921af50 From 42f912e03cf123934367bee5c294418525387b15 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Tue, 21 Oct 2025 13:15:03 -0400 Subject: [PATCH 110/150] Change registry access protected to allocatable=parameter --- src/data/registry.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/data/registry.xml b/src/data/registry.xml index 6aae0893..6299ecf6 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -1535,13 +1535,13 @@ + allocatable="parameter"> .true. + allocatable="parameter"> 1 + units="flag" type="logical" + allocatable="parameter"> .true. .false. From 581148be0e89f0bbd9d65797ae19354c9eafb803 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Tue, 21 Oct 2025 11:30:18 -0600 Subject: [PATCH 111/150] clean up registry --- .gitmodules | 2 +- src/data/registry.xml | 6 ++++-- src/physics/ncar_ccpp | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index c968e015..ec885458 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = b0cf14d631c641c69280f7b7ad81f61cc921af50 + fxtag = d76289441a862ba2e3cbcb9ec27c19fbe97277d9 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/data/registry.xml b/src/data/registry.xml index 9f2580f7..4a15fcc1 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -1746,9 +1746,11 @@ + units="count" type="integer" + allocatable="parameter"> 2 + diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index b0cf14d6..d7628944 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit b0cf14d631c641c69280f7b7ad81f61cc921af50 +Subproject commit d76289441a862ba2e3cbcb9ec27c19fbe97277d9 From 1b4a5bb13ee17f592aba2d31f7e1944e9abd8cfd Mon Sep 17 00:00:00 2001 From: peverwhee Date: Thu, 23 Oct 2025 14:46:42 -0600 Subject: [PATCH 112/150] add missing initial values to registry; removed unused fields --- src/data/registry.xml | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/src/data/registry.xml b/src/data/registry.xml index 4a15fcc1..9c049bbf 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -1686,6 +1686,7 @@ units="1" type="real" kind="kind_phys" allocatable="allocatable"> horizontal_dimension vertical_layer_dimension + 0._kind_phys LAMBDAC pbuf_LAMBDAC horizontal_dimension vertical_layer_dimension + 0._kind_phys MU pbuf_MU horizontal_dimension vertical_layer_dimension + 0._kind_phys ICLWP pbuf_ICLWP horizontal_dimension vertical_layer_dimension + 0._kind_phys ICIWP pbuf_ICIWP horizontal_dimension vertical_layer_dimension + 0._kind_phys ICSWP pbuf_ICSWP horizontal_dimension vertical_layer_dimension + 0._kind_phys DEI pbuf_DEI horizontal_dimension vertical_layer_dimension + 0._kind_phys DES pbuf_DES horizontal_dimension vertical_layer_dimension + 0._kind_phys QRS pbuf_QRS horizontal_dimension vertical_layer_dimension + 0._kind_phys QRL pbuf_QRL 2 - From 99ec7a7b46b0c6d5505e092a1c952a4ec3ff52ba Mon Sep 17 00:00:00 2001 From: peverwhee Date: Thu, 23 Oct 2025 14:51:40 -0600 Subject: [PATCH 113/150] update atmos phys hash --- .gitmodules | 2 +- src/physics/ncar_ccpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index ec885458..0edc23da 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = d76289441a862ba2e3cbcb9ec27c19fbe97277d9 + fxtag = 215a362ede33dc12eaaa289a5601309c01109b47 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index d7628944..215a362e 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit d76289441a862ba2e3cbcb9ec27c19fbe97277d9 +Subproject commit 215a362ede33dc12eaaa289a5601309c01109b47 From c1de14726b8af364ff8a7b2fef17461315ede140 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Thu, 23 Oct 2025 16:11:36 -0600 Subject: [PATCH 114/150] remove unnecessary reordering --- cime_config/cam_autogen.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cime_config/cam_autogen.py b/cime_config/cam_autogen.py index 560f8201..695364b2 100644 --- a/cime_config/cam_autogen.py +++ b/cime_config/cam_autogen.py @@ -489,14 +489,6 @@ def generate_physics_suites(build_cache, preproc_defs, host_name, # End if # End for # End for - - # Reorder scheme files to put namelist files first - namelist_files = [f for f in scheme_files if "_namelist" in f] - other_files = [f for f in scheme_files if "_namelist" not in f] - - # Concatenate with namelist files first - scheme_files = namelist_files + other_files - # Figure out if we need to generate new physics code genccpp_dir = os.path.join(bldroot, "ccpp") kind_phys = ['kind_phys = REAL64'] From 31c9290d5e7dc9edfdfcc2f6a16d4a26a19c3c36 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Thu, 23 Oct 2025 19:53:47 -0600 Subject: [PATCH 115/150] remove code that is no longer necessary --- .gitmodules | 2 +- src/data/cam_var_init_marks.inc | 55 +--------------------------- src/data/cam_var_init_marks_decl.inc | 1 - src/data/generate_registry_data.py | 2 +- src/data/registry.xml | 1 - src/physics/ncar_ccpp | 2 +- 6 files changed, 5 insertions(+), 58 deletions(-) diff --git a/.gitmodules b/.gitmodules index 0edc23da..47d03132 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/peverwhee/atmospheric_physics - fxtag = 215a362ede33dc12eaaa289a5601309c01109b47 + fxtag = 9e8bf60441f3fe718b96c2d42d59f031475225ee fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/data/cam_var_init_marks.inc b/src/data/cam_var_init_marks.inc index dd83c4bd..b9b44c03 100644 --- a/src/data/cam_var_init_marks.inc +++ b/src/data/cam_var_init_marks.inc @@ -67,7 +67,7 @@ end subroutine mark_as_read_from_file - logical function is_initialized(varname, error_on_not_found) + logical function is_initialized(varname) ! This function checks if the variable, , is already ! initialized according to the 'initialized_vars' array. @@ -76,23 +76,15 @@ ! Dummy argument character(len=*), intent(in) :: varname ! Variable name being checked - logical, optional, intent(in) :: error_on_not_found ! Local variables integer :: stdnam_idx ! Standard name array index logical :: found ! Check that was found - logical :: error_on_not_found_loc character(len=*), parameter :: subname = 'is_initialized: ' is_initialized = .false. found = .false. - if (present(error_on_not_found)) then - error_on_not_found_loc = error_on_not_found - else - error_on_not_found_loc = .true. - end if - ! Check if variable is initialized (PARAM, INITIALIZED, or READ_FROM_FILE) do stdnam_idx = 1, phys_var_num if (trim(phys_var_stdnames(stdnam_idx)) == trim(varname)) then @@ -102,7 +94,7 @@ end if end do - if (.not. found .and. error_on_not_found_loc) then + if (.not. found) then ! This condition is an internal error, it should not happen call endrun(subname//": Variable '"//trim(varname)// & "' is missing from phys_var_stdnames array.") @@ -110,49 +102,6 @@ end function is_initialized - logical function is_parameter(varname, error_on_not_found) - - ! This function checks if the variable, , is - ! a parameter according to the 'initialized_vars' array. - - use cam_abortutils, only: endrun - - ! Dummy argument - character(len=*), intent(in) :: varname ! Variable name being checked - logical, optional, intent(in) :: error_on_not_found - - ! Local variables - integer :: stdnam_idx ! Standard name array index - logical :: found ! Check that was found - logical :: error_on_not_found_loc - character(len=*), parameter :: subname = 'is_parameter: ' - - is_parameter = .false. - found = .false. - - if (present(error_on_not_found)) then - error_on_not_found_loc = error_on_not_found - else - error_on_not_found_loc = .true. - end if - - ! Check if variable is a parameter - do stdnam_idx = 1, phys_var_num - if (trim(phys_var_stdnames(stdnam_idx)) == trim(varname)) then - is_parameter = (initialized_vars(stdnam_idx) == PARAM) - found = .true. - exit ! Exit loop once variable has been found and checked - end if - end do - - if (.not. found .and. error_on_not_found_loc) then - ! This condition is an internal error, it should not happen - call endrun(subname//": Variable '"//trim(varname)// & - "' is missing from phys_var_stdnames array.") - end if - - end function is_parameter - subroutine is_read_from_file(varname, is_read, stdnam_idx_out) ! This subroutine checks if the variable, , is read from diff --git a/src/data/cam_var_init_marks_decl.inc b/src/data/cam_var_init_marks_decl.inc index 1e6ac19f..dbbe5a87 100644 --- a/src/data/cam_var_init_marks_decl.inc +++ b/src/data/cam_var_init_marks_decl.inc @@ -2,7 +2,6 @@ public :: mark_as_initialized public :: mark_as_read_from_file public :: is_initialized - public :: is_parameter public :: is_read_from_file !! Parameterized initialized_vars options - order matters diff --git a/src/data/generate_registry_data.py b/src/data/generate_registry_data.py index e2efdce6..30c09f4a 100755 --- a/src/data/generate_registry_data.py +++ b/src/data/generate_registry_data.py @@ -306,7 +306,7 @@ def write_initial_value(self, outfile, indent, init_var, ddt_str, physconst_vars raise TypeError(emsg.format(var_name, self.var_type)) # end if elif init_val: - outfile.write(f"if ({init_var} .and. (.not. is_initialized('{self.standard_name}', error_on_not_found=.false.) .or. is_parameter('{self.standard_name}', error_on_not_found=.false.))) then", indent) + outfile.write(f"if ({init_var}) then", indent) outfile.write(f"{var_name} = {init_val}", indent+1) if self.initial_val_vars and self.initial_val_vars.issubset(physconst_vars): outfile.write(f"call mark_as_initialized('{self.standard_name}')", indent+1) diff --git a/src/data/registry.xml b/src/data/registry.xml index 9c049bbf..b6fdd60a 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -36,7 +36,6 @@ - diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 215a362e..9e8bf604 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 215a362ede33dc12eaaa289a5601309c01109b47 +Subproject commit 9e8bf60441f3fe718b96c2d42d59f031475225ee From 9098e4cc04ed4177e55ef32aab5ad713b3f5677a Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 24 Oct 2025 13:45:05 -0600 Subject: [PATCH 116/150] Bring in CAM4 physics test atmospheric_physics branch. --- .gitmodules | 4 ++-- src/physics/ncar_ccpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 5d1800af..dd98a1ed 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,8 +19,8 @@ fxDONOTUSEurl = https://github.com/MPAS-Dev/MPAS-Model.git [submodule "ncar-physics"] path = src/physics/ncar_ccpp - url = https://github.com/ESCOMP/atmospheric_physics - fxtag = 73fce2706c1f70e1ba63ff4bb3dcb80323bb9cc6 + url = https://github.com/nusbaume/atmospheric_physics + fxtag = 3223840e45ba991030a9099dee165be01740b526 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 73fce270..3223840e 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 73fce2706c1f70e1ba63ff4bb3dcb80323bb9cc6 +Subproject commit 3223840e45ba991030a9099dee165be01740b526 From 05cf588c4f10f69f771c6105e3e269b8a17cdbb4 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 24 Oct 2025 14:29:27 -0600 Subject: [PATCH 117/150] Add 'usermods_dirs' directories with needed aquaplanet user_nl_cpl changes. --- cime_config/config_component.xml | 6 ++--- cime_config/usermods_dirs/aquap/user_nl_cpl | 28 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 cime_config/usermods_dirs/aquap/user_nl_cpl diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 6fcca200..6fd2ddd8 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -292,9 +292,9 @@ - $SRCROOT/components/cam/cime_config/usermods_dirs/aquap - $SRCROOT/components/cam/cime_config/usermods_dirs/aquap - $SRCROOT/components/cam/cime_config/usermods_dirs/scam_mandatory + $COMP_ROOT_DIR_ATM/cime_config/usermods_dirs/aquap + $COMP_ROOT_DIR_ATM/cime_config/usermods_dirs/aquap + run_component_cam env_case.xml diff --git a/cime_config/usermods_dirs/aquap/user_nl_cpl b/cime_config/usermods_dirs/aquap/user_nl_cpl new file mode 100644 index 00000000..64e32725 --- /dev/null +++ b/cime_config/usermods_dirs/aquap/user_nl_cpl @@ -0,0 +1,28 @@ +!------------------------------------------------------------------------ +! Users should ONLY USE user_nl_cpl to change namelists variables +! for namelist variables in drv_in (except for the ones below) and +! any keyword/values in seq_maps.rc +! Users should add ALL user specific namelist and seq_maps.rc changes below +! using the following syntax +! namelist_var = new_namelist_value +! or +! mapname = new_map_name +! For example to change the default value of ocn2atm_fmapname to 'foo' use +! ocn2atm_fmapname = 'foo' +! +! Note that some namelist variables MAY NOT be changed in user_nl_cpl - +! they are defined in a $CASEROOT xml file and must be changed with +! xmlchange. +! +! For example, rather than set username to 'foo' in user_nl_cpl, call +! ./xmlchange USER=foo +!------------------------------------------------------------------------ +! +! SPECIAL NOTE FOR AQUAPLANET +! Do not modify any of the following orb_ entries +! co2vmr is not specified via the namelist but rather in the env_run.xml +! +orb_eccen = 0. +orb_obliq = 0. +orb_mvelp = 0. +orb_mode = "fixed_parameters" From 4a00f225bb22c245818755f506e04b74efaf0f7a Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 27 Oct 2025 09:32:50 -0600 Subject: [PATCH 118/150] Add 'aquaplanet' config flag, remove un-used 'ocn' config option, and update namelist to set proper 'bnd_topo' value. --- cime_config/cam_config.py | 42 ++++++++++++++----------- cime_config/config_component.xml | 4 +-- cime_config/namelist_definition_cam.xml | 4 +++ 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/cime_config/cam_config.py b/cime_config/cam_config.py index 20e3b3f5..fc0ddc9b 100644 --- a/cime_config/cam_config.py +++ b/cime_config/cam_config.py @@ -481,32 +481,32 @@ def __init__(self, case, case_log): self.create_config("analytic_ic", analy_ic_desc, analy_ic_val, [0, 1], is_nml_attr=True) - #-------------------- - # Set ocean component - #-------------------- + #-------------------------- + # Check if running an + # aquaplanet configuration + #-------------------------- + + if user_config_opts.aquaplanet: + aquap_flag = 1 + else: + aquap_flag = 0 - ocn_valid_vals = ["docn", "dom", "som", "socn", - "aquaplanet", "pop", "mom"] + aquap_desc = ["Switch to use aquaplanet configuration: ", + "0 => no ", + "1 => yes."] - ocn_desc = ["The ocean model being used.", - "Valid values include prognostic ocean models (POP or MOM),", - "data ocean models (DOCN or DOM), a stub ocean (SOCN), ", - "and an aqua planet ocean (aquaplanet).", - "This does not impact how the case is built, only how", - "attributes are matched when searching for namelist defaults."] + self.create_config("aquaplanet", aquap_desc, + aquap_flag, [0, 1], is_nml_attr=True) - self.create_config("ocn", ocn_desc, comp_ocn, - ocn_valid_vals, is_nml_attr=True) + #-------------------------- + # Set physics_suites string + #-------------------------- phys_desc = ["A semicolon-separated list of physics suite definition " "file (SDF) names.", "To specify the Kessler and Held-Suarez suites as ", "run time options, use '--physics-suites kessler;held_suarez_1994'."] - #-------------------------- - # Set physics_suites string - #-------------------------- - self.create_config("physics_suites", phys_desc, user_config_opts.physics_suites) @@ -627,11 +627,17 @@ def parse_config_opts(cls, config_opts, test_mode=False): default="", help="""Name of dycore""") + parser.add_argument("--aquaplanet", + action='store_true', + required=False, + help="""Flag to turn on aquaplanet + settings (0 = False, 1 = True).""") + parser.add_argument("--analytic-ic", action='store_true', required=False, help="""Flag to turn on Analytic Initial - Conditions (ICs).""") + Conditions (ICs). 0 = False, 1 = True.""") parser.add_argument("--dyn-kind", type=str, diff --git a/cime_config/config_component.xml b/cime_config/config_component.xml index 6fd2ddd8..c0f6d5db 100644 --- a/cime_config/config_component.xml +++ b/cime_config/config_component.xml @@ -169,8 +169,8 @@ --dyn none --physics-suites adiabatic - + --aquaplanet + --aquaplanet diff --git a/cime_config/namelist_definition_cam.xml b/cime_config/namelist_definition_cam.xml index 2966f6f5..88a72945 100644 --- a/cime_config/namelist_definition_cam.xml +++ b/cime_config/namelist_definition_cam.xml @@ -253,6 +253,10 @@ UNSET_PATH UNSET_PATH UNSET_PATH + UNSET_PATH + UNSET_PATH + UNSET_PATH + UNSET_PATH From 5cd4a1d836793e2dc3d6fb62e0214e6896be9ec1 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 29 Oct 2025 11:14:56 -0600 Subject: [PATCH 119/150] Update 'physconst' namelist values for a CAM4 aquaplanet configuration. --- src/data/namelist_definition_physconst.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/data/namelist_definition_physconst.xml b/src/data/namelist_definition_physconst.xml index c924b846..7c3e99d5 100644 --- a/src/data/namelist_definition_physconst.xml +++ b/src/data/namelist_definition_physconst.xml @@ -15,6 +15,7 @@ 0.0 + 9.79764 @@ -27,6 +28,7 @@ 0.0 + 86164.10063718943 @@ -39,6 +41,7 @@ 0.0 + 18.01618112892741 @@ -51,6 +54,7 @@ 0.0 + 1.846e3 @@ -75,6 +79,7 @@ 0.0 + 28.96623324623746 @@ -87,6 +92,7 @@ 0.0 + 6.37100e6 From 7fb46a0ee59e0e92c39262515dd299ae3a5b2659 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 29 Oct 2025 15:55:19 -0600 Subject: [PATCH 120/150] Update ncar_ccpp submodule to bring in needed physics namelist mods for CAM4 aquaplanet simulations. --- .gitmodules | 2 +- src/physics/ncar_ccpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index e9508ee3..ae7a9a8c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/nusbaume/atmospheric_physics - fxtag = 3223840e45ba991030a9099dee165be01740b526 + fxtag = 2824bb218f7ca5ecd0d0d00cca61ff54a8f7ac11 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 3223840e..2824bb21 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 3223840e45ba991030a9099dee165be01740b526 +Subproject commit 2824bb218f7ca5ecd0d0d00cca61ff54a8f7ac11 From 10008b1113d857d5c8cb968f669c0ba6ac568121 Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Mon, 3 Nov 2025 11:48:52 -0500 Subject: [PATCH 121/150] Address review comments --- src/data/registry.xml | 1 + src/dynamics/se/dp_coupling.F90 | 57 ++++++++++--------- src/dynamics/se/gravity_waves_sources.F90 | 48 +++++++++------- .../utils/gravity_wave_drag_ridge_read.F90 | 1 - src/physics/utils/tropopause_climo_read.F90 | 1 - 5 files changed, 57 insertions(+), 51 deletions(-) diff --git a/src/data/registry.xml b/src/data/registry.xml index 6299ecf6..ed41e734 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -21,6 +21,7 @@ $SRCROOT/src/data/ref_pres.meta $SRCROOT/src/dynamics/utils/vert_coord.meta $SRCROOT/src/dynamics/utils/hycoef.meta + diff --git a/src/dynamics/se/dp_coupling.F90 b/src/dynamics/se/dp_coupling.F90 index 5eb9f981..de4f7fe0 100644 --- a/src/dynamics/se/dp_coupling.F90 +++ b/src/dynamics/se/dp_coupling.F90 @@ -65,6 +65,8 @@ subroutine d_p_coupling(cam_runtime_opts, phys_state, phys_tend, dyn_out) use time_mod, only: timelevel_qdp use control_mod, only: qsplit + use shr_kind_mod, only: shr_kind_cl + ! arguments type(runtime_options), intent(in) :: cam_runtime_opts ! Runtime settings object type(dyn_export_t), intent(inout) :: dyn_out ! dynamics export @@ -106,6 +108,7 @@ subroutine d_p_coupling(cam_runtime_opts, phys_state, phys_tend, dyn_out) character(len=*), parameter :: subname = 'd_p_coupling' character(len=200) :: stdname_test + character(len=shr_kind_cl) :: errmsg !---------------------------------------------------------------------------- @@ -120,9 +123,9 @@ subroutine d_p_coupling(cam_runtime_opts, phys_state, phys_tend, dyn_out) if (fv_nphys > 0) then nphys = fv_nphys else - allocate(qgll(np,np,nlev,num_advected), stat=ierr) + allocate(qgll(np,np,nlev,num_advected), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'qgll(np,np,nlev,num_advected)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) nphys = np end if @@ -130,49 +133,49 @@ subroutine d_p_coupling(cam_runtime_opts, phys_state, phys_tend, dyn_out) const_data_ptr => cam_constituents_array() ! Allocate temporary arrays to hold data for physics decomposition - allocate(ps_tmp(nphys_pts,nelemd), stat=ierr) + allocate(ps_tmp(nphys_pts,nelemd), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'ps_tmp(nphys_pts,nelemd)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) - allocate(dp3d_tmp(nphys_pts,pver,nelemd), stat=ierr) + allocate(dp3d_tmp(nphys_pts,pver,nelemd), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'dp3d_tmp(nphys_pts,pver,nelemd)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) - allocate(dp3d_tmp_tmp(nphys_pts,pver), stat=ierr) + allocate(dp3d_tmp_tmp(nphys_pts,pver), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'dp3d_tmp_tmp(nphys_pts,pver)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) - allocate(phis_tmp(nphys_pts,nelemd), stat=ierr) + allocate(phis_tmp(nphys_pts,nelemd), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'phis_tmp(nphys_pts,nelemd)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) - allocate(T_tmp(nphys_pts,pver,nelemd), stat=ierr) + allocate(T_tmp(nphys_pts,pver,nelemd), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'T_tmp(nphys_pts,pver,nelemd)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) - allocate(uv_tmp(nphys_pts,2,pver,nelemd), stat=ierr) + allocate(uv_tmp(nphys_pts,2,pver,nelemd), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'uv_tmp(nphys_pts,2,pver,nelemd)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) - allocate(q_tmp(nphys_pts,pver,num_advected,nelemd), stat=ierr) + allocate(q_tmp(nphys_pts,pver,num_advected,nelemd), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'q_tmp(nphys_pts,pver,num_advected,nelemd)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) - allocate(omega_tmp(nphys_pts,pver,nelemd), stat=ierr) + allocate(omega_tmp(nphys_pts,pver,nelemd), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'omega_tmp(nphys_pts,pver,nelemd)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) - allocate(frontgf(nphys_pts,pver,nelemd), stat=ierr) + allocate(frontgf(nphys_pts,pver,nelemd), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'frontgf(nphys_pts,pver,nelemd)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) - allocate(frontga(nphys_pts,pver,nelemd), stat=ierr) + allocate(frontga(nphys_pts,pver,nelemd), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'frontga(nphys_pts,pver,nelemd)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) - allocate(vort4gw(nphys_pts,pver,nelemd), stat=ierr) + allocate(vort4gw(nphys_pts,pver,nelemd), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'vort4gw(nphys_pts,pver,nelemd)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) if (iam < par%nprocs) then @@ -243,9 +246,9 @@ subroutine d_p_coupling(cam_runtime_opts, phys_state, phys_tend, dyn_out) omega_tmp(:,:,:) = 0._r8 phis_tmp(:,:) = 0._r8 q_tmp(:,:,:,:) = 0._r8 - frontgf(:,:,:) = 0._r8 - frontga(:,:,:) = 0._r8 - vort4gw(:,:,:) = 0._r8 + frontgf(:,:,:) = 0._r8 + frontga(:,:,:) = 0._r8 + vort4gw(:,:,:) = 0._r8 endif ! iam < par%nprocs diff --git a/src/dynamics/se/gravity_waves_sources.F90 b/src/dynamics/se/gravity_waves_sources.F90 index 27ef3a6a..5a4cb075 100644 --- a/src/dynamics/se/gravity_waves_sources.F90 +++ b/src/dynamics/se/gravity_waves_sources.F90 @@ -55,6 +55,7 @@ subroutine gws_src_fnct(elem, tl, tlq, frontgf, frontga,nphys) use vert_coord, only: pver use cam_abortutils, only: check_allocate + use shr_kind_mod, only: shr_kind_cl !SE dycore: use derivative_mod, only: derivinit @@ -73,8 +74,9 @@ subroutine gws_src_fnct(elem, tl, tlq, frontgf, frontga,nphys) ! Local variables type (hybrid_t) :: hybrid integer :: nets, nete, ithr, ncols, ie, iret - real(kind=r8), allocatable :: frontgf_thr(:,:,:,:) - real(kind=r8), allocatable :: frontga_thr(:,:,:,:) + real(kind=r8), allocatable :: frontgf_thr(:,:,:,:) + real(kind=r8), allocatable :: frontga_thr(:,:,:,:) + character(len=shr_kind_cl) :: errmsg character(len=*), parameter :: subname = 'gws_src_fnct' @@ -85,15 +87,15 @@ subroutine gws_src_fnct(elem, tl, tlq, frontgf, frontga,nphys) hybrid = config_thread_region(par,'serial') call get_loop_ranges(hybrid,ibeg=nets,iend=nete) - allocate(frontgf_thr(nphys,nphys,nlev,nets:nete), stat=iret) + allocate(frontgf_thr(nphys,nphys,nlev,nets:nete), stat=iret, errmsg=errmsg) call check_allocate(iret, subname, & 'frontgf_thr(nphys,nphys,nlev,nets:nete)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) - allocate(frontga_thr(nphys,nphys,nlev,nets:nete), stat=iret) + allocate(frontga_thr(nphys,nphys,nlev,nets:nete), stat=iret, errmsg=errmsg) call check_allocate(iret, subname, & 'frontga_thr(nphys,nphys,nlev,nets:nete)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) call compute_frontogenesis(frontgf_thr,frontga_thr,tl,tlq,elem,deriv,hybrid,nets,nete,nphys) if (fv_nphys>0) then @@ -121,10 +123,11 @@ subroutine gws_src_vort(elem, tl, tlq, vort4gw, nphys) use dof_mod, only : UniquePoints use hybrid_mod, only : config_thread_region, get_loop_ranges use parallel_mod, only : par - use ppgrid, only : pver + use vert_coord, only : pver use thread_mod, only : horz_num_threads use dimensions_mod, only : fv_nphys use cam_abortutils, only : check_allocate + use shr_kind_mod, only : shr_kind_cl implicit none type (element_t), intent(in), dimension(:) :: elem @@ -139,6 +142,7 @@ subroutine gws_src_vort(elem, tl, tlq, vort4gw, nphys) ! real(kind=r8), allocatable :: vort4gw_thr(:,:,:,:) + character(len=shr_kind_cl) :: errmsg character(len=*), parameter :: subname = 'gws_src_vort' @@ -148,10 +152,10 @@ subroutine gws_src_vort(elem, tl, tlq, vort4gw, nphys) hybrid = config_thread_region(par,'serial') call get_loop_ranges(hybrid,ibeg=nets,iend=nete) - allocate(vort4gw_thr(nphys,nphys,nlev,nets:nete), stat=ierr) + allocate(vort4gw_thr(nphys,nphys,nlev,nets:nete), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, & 'vort4gw_thr(nphys,nphys,nlev,nets:nete)', & - file=__FILE__, line=__LINE__) + file=__FILE__, line=__LINE__, errmsg=errmsg) call compute_vorticity_4gw(vort4gw_thr,tl,tlq,elem,deriv,hybrid,nets,nete,nphys) @@ -202,7 +206,7 @@ subroutine compute_vorticity_4gw(vort4gw,tl,tlq,elem,ederiv,hybrid,nets,nete,nph !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! First calculate vorticity on GLL grid !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! set timelevel=1 fro velocities + ! set timelevel=1 for velocities n0=tl do ie=nets,nete do k=1,nlev @@ -213,7 +217,7 @@ subroutine compute_vorticity_4gw(vort4gw,tl,tlq,elem,ederiv,hybrid,nets,nete,nph end do ! pack call edgeVpack(edge1, vort_gll(:,:,:,ie),nlev,0,ie) - enddo + end do call bndry_exchange(hybrid,edge1,location='compute_vorticity_4gw') do ie=nets,nete call edgeVunpack(edge1, vort_gll(:,:,:,ie),nlev,0,ie) @@ -238,7 +242,7 @@ subroutine compute_vorticity_4gw(vort4gw,tl,tlq,elem,ederiv,hybrid,nets,nete,nph vort4gw(:,:,k,ie) = vort_gll(:,:,k,ie) end do end if - enddo + end do end subroutine compute_vorticity_4gw @@ -292,9 +296,9 @@ subroutine compute_frontogenesis(frontgf,frontga,tl,tlq,elem,ederiv,hybrid,nets, real(r8) :: gradp(np,np,2) ! grad(pressure) real(r8) :: theta(np,np,nlev) ! potential temperature at mid points real(r8) :: dtheta_dp(np,np,nlev) ! d(theta)/dp for eta to pressure surface correction - real(r8) :: dum_grad(np,np,2) ! horizontal gradient of zonal and meridional wind on cartesian coordinate on isobaric surface - real(r8) :: dum_cart(np,np,3,nlev) ! zonal & meridional wind on cartesian coordinate - real(r8) :: ddp_dum_cart(np,np,3,nlev) ! vertical gradient of zonal & meridional wind on cartesian coordinate + real(r8) :: grad_wind_cart(np,np,2) ! horizontal gradient of zonal and meridional wind on cartesian coordinate on isobaric surface + real(r8) :: wind_cart(np,np,3,nlev) ! zonal & meridional wind on cartesian coordinate + real(r8) :: ddp_wind_cart(np,np,3,nlev) ! vertical gradient of zonal & meridional wind on cartesian coordinate real(r8) :: C(np,np,2), sum_water(np,np) ! By Mark Taylor @@ -315,7 +319,7 @@ subroutine compute_frontogenesis(frontgf,frontga,tl,tlq,elem,ederiv,hybrid,nets, ! Each of these gradients is represented in *spherical* coordinates (i=1,2) ! ! We then dot each of these vectors with grad(theta). This dot product is computed - ! in spherical coordinates. The end result is dum_cart(c), for c=1,2,3 + ! in spherical coordinates. The end result is wind_cart(c), for c=1,2,3 ! These three scalars are the three Cartesian coefficients of ! the vector "grad(theta)*grad(v)" ! @@ -357,26 +361,26 @@ subroutine compute_frontogenesis(frontgf,frontga,tl,tlq,elem,ederiv,hybrid,nets, do k=1,nlev do component=1,3 - dum_cart(:,:,component,k) = sum( elem(ie)%vec_sphere2cart(:,:,component,:) * elem(ie)%state%v(:,:,:,k,tl),3 ) + wind_cart(:,:,component,k) = sum( elem(ie)%vec_sphere2cart(:,:,component,:) * elem(ie)%state%v(:,:,:,k,tl),3 ) end do end do do component=1,3 - call compute_vertical_derivative(pint,p,dum_cart(:,:,component,:),ddp_dum_cart(:,:,component,:)) + call compute_vertical_derivative(pint,p,wind_cart(:,:,component,:),ddp_wind_cart(:,:,component,:)) end do do k=1,nlev call gradient_sphere(p(:,:,k),ederiv,elem(ie)%Dinv,gradp) do component=1,3 - call gradient_sphere(dum_cart(:,:,component,k),ederiv,elem(ie)%Dinv,dum_grad) + call gradient_sphere(wind_cart(:,:,component,k),ederiv,elem(ie)%Dinv,grad_wind_cart) do i=1,2 - dum_grad(:,:,i) = dum_grad(:,:,i) - ddp_dum_cart(:,:,component,k) * gradp(:,:,i) + grad_wind_cart(:,:,i) = grad_wind_cart(:,:,i) - ddp_wind_cart(:,:,component,k) * gradp(:,:,i) end do - dum_cart(:,:,component,k) = sum( gradth(:,:,:,k,ie) * dum_grad , 3 ) + wind_cart(:,:,component,k) = sum( gradth(:,:,:,k,ie) * grad_wind_cart , 3 ) end do do i=1,2 - C(:,:,i) = sum(dum_cart(:,:,:,k)*elem(ie)%vec_sphere2cart(:,:,:,i), 3) + C(:,:,i) = sum(wind_cart(:,:,:,k)*elem(ie)%vec_sphere2cart(:,:,:,i), 3) end do ! gradth dot C diff --git a/src/physics/utils/gravity_wave_drag_ridge_read.F90 b/src/physics/utils/gravity_wave_drag_ridge_read.F90 index b329d89e..8694d867 100644 --- a/src/physics/utils/gravity_wave_drag_ridge_read.F90 +++ b/src/physics/utils/gravity_wave_drag_ridge_read.F90 @@ -15,7 +15,6 @@ module gravity_wave_drag_ridge_read implicit none private - save public :: gravity_wave_drag_ridge_read_readnl public :: gravity_wave_drag_ridge_read_file diff --git a/src/physics/utils/tropopause_climo_read.F90 b/src/physics/utils/tropopause_climo_read.F90 index 87af3d08..f423b9a7 100644 --- a/src/physics/utils/tropopause_climo_read.F90 +++ b/src/physics/utils/tropopause_climo_read.F90 @@ -13,7 +13,6 @@ module tropopause_climo_read implicit none private - save public :: tropopause_climo_readnl public :: tropopause_climo_read_file From 7368353313e53c1e82fea5c5c969e3c6822ce8de Mon Sep 17 00:00:00 2001 From: Haipeng Lin Date: Mon, 3 Nov 2025 11:49:12 -0500 Subject: [PATCH 122/150] Update src/physics/utils/gravity_wave_drag_ridge_read.F90 Co-authored-by: Jesse Nusbaumer --- src/physics/utils/gravity_wave_drag_ridge_read.F90 | 1 + 1 file changed, 1 insertion(+) diff --git a/src/physics/utils/gravity_wave_drag_ridge_read.F90 b/src/physics/utils/gravity_wave_drag_ridge_read.F90 index 8694d867..61ec571a 100644 --- a/src/physics/utils/gravity_wave_drag_ridge_read.F90 +++ b/src/physics/utils/gravity_wave_drag_ridge_read.F90 @@ -345,6 +345,7 @@ subroutine gravity_wave_drag_ridge_read_file() call cam_pio_closefile(fh_rdggm) deallocate(fh_rdggm) + nullify(fh_rdggm) ! Mark variables as initialized so they are not read from ic file. call mark_as_initialized('grid_box_area_for_gamma_ridge_gravity_wave_drag') From 4f57c9f8bf9d96079faced0dced00159d22d9250 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 6 Nov 2025 09:37:56 -0700 Subject: [PATCH 123/150] Update atmospheric_physics submodule to bring in new radiative constituents scheme. --- .gitmodules | 2 +- src/physics/ncar_ccpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index ae7a9a8c..a9cd1693 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/nusbaume/atmospheric_physics - fxtag = 2824bb218f7ca5ecd0d0d00cca61ff54a8f7ac11 + fxtag = 9bc6eff5100874b26af8aa070d11c75591388e65 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 2824bb21..9bc6eff5 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 2824bb218f7ca5ecd0d0d00cca61ff54a8f7ac11 +Subproject commit 9bc6eff5100874b26af8aa070d11c75591388e65 From d428a56eef98f440954eb1c584280858b413befe Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 6 Nov 2025 11:12:03 -0700 Subject: [PATCH 124/150] Update atmospheric_physics submodule to fix source code bugs. --- .gitmodules | 2 +- src/physics/ncar_ccpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index a9cd1693..37e6f3cd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/nusbaume/atmospheric_physics - fxtag = 9bc6eff5100874b26af8aa070d11c75591388e65 + fxtag = 547c61db025d6a075ae9217406e20bc8d50033b5 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 9bc6eff5..547c61db 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 9bc6eff5100874b26af8aa070d11c75591388e65 +Subproject commit 547c61db025d6a075ae9217406e20bc8d50033b5 From 315456f40e1fbcc03c27c241fbb06ea0a08b058d Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 6 Nov 2025 12:01:59 -0700 Subject: [PATCH 125/150] Fix bug found during QPC4 experiment. --- src/dynamics/se/gravity_waves_sources.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dynamics/se/gravity_waves_sources.F90 b/src/dynamics/se/gravity_waves_sources.F90 index 5a4cb075..0591c3ef 100644 --- a/src/dynamics/se/gravity_waves_sources.F90 +++ b/src/dynamics/se/gravity_waves_sources.F90 @@ -344,7 +344,7 @@ subroutine compute_frontogenesis(frontgf,frontga,tl,tlq,elem,ederiv,hybrid,nets, ! moist pressure at interface for next iteration pint(:,:,k+1) = pint(:,:,k)+elem(ie)%state%dp3d(:,:,k,tl) ! - theta(:,:) = elem(ie)%state%T(:,:,k,tl)*(psurf_ref / p(:,:,k))**cappa + theta(:,:,k) = elem(ie)%state%T(:,:,k,tl)*(psurf_ref / p(:,:,k))**cappa end do call compute_vertical_derivative(pint,p,theta,dtheta_dp) From 467b911f6db889254f3b72bb8e9d495f9ca23e7a Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 7 Nov 2025 16:06:11 -0700 Subject: [PATCH 126/150] Fix variable string length bug in PIO reader. --- src/physics/utils/pio_reader.F90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/physics/utils/pio_reader.F90 b/src/physics/utils/pio_reader.F90 index 23248230..f9ef6637 100644 --- a/src/physics/utils/pio_reader.F90 +++ b/src/physics/utils/pio_reader.F90 @@ -2416,10 +2416,10 @@ subroutine get_pio_errmsg(caller_errcode, varname, errcode, errmsg, file_msg) errcode = pio_get_msg_err else if (file_msg_flag) then - write(errmsg, '(a,a,a,a)') "Error for file '", varname, "' - message: ", trim(pio_error) + write(errmsg, '(a,a,a,a)') "Error for file '", trim(varname), "' - message: ", trim(pio_error) else !Variable error message - write(errmsg, '(a,a,a,a)') "Error for variable '", varname, "' - message: ", trim(pio_error) + write(errmsg, '(a,a,a,a)') "Error for variable '", trim(varname), "' - message: ", trim(pio_error) end if errcode = caller_errcode end if @@ -2546,7 +2546,7 @@ subroutine var_subset_check(varname, var_ndims, dim_sizes, do_subset, alloc_dims end if !If both start and count are not present, - !then set alloc_dims to have the same + !then set alloc_dims to have the same !dimensionality as the file variable: if (.not. present(start) .and. .not. present(count)) then allocate(alloc_dims, source=dim_sizes, stat=errcode, errmsg=errmsg) From d70779608dcdfcd9df0a8785b82cba14dded922e Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 11 Nov 2025 08:07:54 -0700 Subject: [PATCH 127/150] Update atmos_phys external with RRTMGP register-phase fixes. --- .gitmodules | 2 +- src/physics/ncar_ccpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 37e6f3cd..80b7b318 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/nusbaume/atmospheric_physics - fxtag = 547c61db025d6a075ae9217406e20bc8d50033b5 + fxtag = 9011861b0534819fa798ae5eb745c0573c5dad22 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 547c61db..9011861b 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 547c61db025d6a075ae9217406e20bc8d50033b5 +Subproject commit 9011861b0534819fa798ae5eb745c0573c5dad22 From 66b786a76b51ba9a6b1177f6a6bf56078eb10742 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 12 Nov 2025 12:23:14 -0700 Subject: [PATCH 128/150] Fix bug in model grid initialization, and add temporary MPAS grid. --- cime_config/namelist_definition_cam.xml | 47 ++++++++++--------------- src/cpl/nuopc/atm_comp_nuopc.F90 | 47 +++++++++++++++++++------ 2 files changed, 55 insertions(+), 39 deletions(-) diff --git a/cime_config/namelist_definition_cam.xml b/cime_config/namelist_definition_cam.xml index 88a72945..fa4440e4 100644 --- a/cime_config/namelist_definition_cam.xml +++ b/cime_config/namelist_definition_cam.xml @@ -14,9 +14,13 @@ UNSET_PATH + + ${DIN_LOC_ROOT}/atm/cam/inic/cam_vcoords_L26_c180105.nc ${DIN_LOC_ROOT}/atm/cam/inic/cam_vcoords_L30_c180105.nc ${DIN_LOC_ROOT}/atm/cam/inic/cam_vcoords_L32_c180105.nc + + ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-01-01_0.23x0.31_L26_c100513.nc ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-09-01_0.23x0.31_L26_c061106.nc ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_1980-01-01_0.47x0.63_L26_c071226.nc @@ -87,23 +91,8 @@ ${DIN_LOC_ROOT}/atm/waccm/ic/FC6X2000_f05_spinup01.cam.i.0002-01-01-00000_c190711.nc ${DIN_LOC_ROOT}/atm/waccm/ic/waccmx_mam4_aqua_4x5_L130_c180803.nc ${DIN_LOC_ROOT}/atm/waccm/ic/waccmx_mam4_aqua_1.9x2.5_L130_c180803.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/T341clim01.cam2.i.0024-01-01-00000.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-01-01_256x512_L26_c030918.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-01-01_128x256_L26_c030918.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-09-01_128x256_L26_c040422.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-01-01_64x128_T42_L26_c031110.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-09-01_64x128_L26_c030918.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-01-01_64x128_L30_c090102.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-09-01_64x128_L30_c031210.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-01-01_64x128_L32_c170510.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-01-01_48x96_L26_c091218.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-09-01_48x96_L26_c040420.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-01-01_48x96_L30_c100426.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-09-01_32x64_L26_c030918.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-01-01_32x64_L30_c090107.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-01-01_8x16_L26_c030228.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-09-01_8x16_L26_c030918.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-01-01_8x16_L30_c090102.nc + + ${DIN_LOC_ROOT}/atm/cam/inic/se/FCts4MTHIST_ne3pg3_spinup02.cam.i.1980-01-01_c240702.nc ${DIN_LOC_ROOT}/atm/cam/inic/se/cam6_QPC6_topo_ne3pg3_mg37_L32_01-01-31_c221214.nc ${DIN_LOC_ROOT}/atm/cam/inic/se/CESM2.F2000climo.ne3np4.cam.i.0003-09-01-00000.nc @@ -148,17 +137,19 @@ ${DIN_LOC_ROOT}/atm/waccm/ic/wa3_ne5np4_1950_spinup.cam2.i.1960-01-01-00000_c150810.nc ${DIN_LOC_ROOT}/atm/waccm/ic/waccm5_1850_ne30np4_L70_0001-01-11-00000_c151217.nc ${DIN_LOC_ROOT}/atm/waccm/ic/fw2000_ne30np4_L70_c181221.nc - ${DIN_LOC_ROOT}/atm/cam/inic/gaus/cami_0000-09-01_64x128_L30_c031210.nc - ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa480_L32_notopo_coords_c240507.nc - ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa120_L32_notopo_coords_c240507.nc - ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa60_L32_notopo_coords_c240507.nc - ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa30_L32_notopo_coords_c240507.nc - ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa480_L58_notopo_coords_c240814.nc - ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa120_L58_notopo_coords_c240814.nc - ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa60_L58_notopo_coords_c240814.nc - ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa480_L93_notopo_coords_c240814.nc - ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa120_L93_notopo_coords_c240814.nc - ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa60_L93_notopo_coords_c240814.nc + + + /glade/campaign/cgd/amp/aherring/mpas-uniform/mpasa120/grids/mpasa120_L26_notopo_coords_c251105.nc + ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa480_L32_notopo_coords_c240507.nc + ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa120_L32_notopo_coords_c240507.nc + ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa60_L32_notopo_coords_c240507.nc + ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa30_L32_notopo_coords_c240507.nc + ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa480_L58_notopo_coords_c240814.nc + ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa120_L58_notopo_coords_c240814.nc + ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa60_L58_notopo_coords_c240814.nc + ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa480_L93_notopo_coords_c240814.nc + ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa120_L93_notopo_coords_c240814.nc + ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa60_L93_notopo_coords_c240814.nc diff --git a/src/cpl/nuopc/atm_comp_nuopc.F90 b/src/cpl/nuopc/atm_comp_nuopc.F90 index b13cbe94..245ee869 100644 --- a/src/cpl/nuopc/atm_comp_nuopc.F90 +++ b/src/cpl/nuopc/atm_comp_nuopc.F90 @@ -56,7 +56,6 @@ module atm_comp_nuopc use perf_mod , only : t_startf, t_stopf use physics_grid , only : global_index_p, get_rlon_all_p, get_rlat_all_p use physics_grid , only : ngcols => num_global_phys_cols - use physics_grid , only : lsize => columns_on_task use physics_grid , only : hdim1_d, hdim2_d use cam_control_mod , only : cam_ctrl_set_orbit use cam_pio_utils , only : cam_pio_createfile, cam_pio_openfile, cam_pio_closefile, pio_subsystem @@ -328,7 +327,8 @@ end subroutine InitializeAdvertise !=============================================================================== subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) - use ESMF, only : ESMF_VMGet + use ESMF, only : ESMF_VMGet + use physics_grid, only : lsize => columns_on_task ! input/output variables type(ESMF_GridComp) :: gcomp @@ -349,10 +349,10 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) integer :: spatialDim integer :: numOwnedElements real(r8), allocatable :: ownedElemCoords(:) - real(r8) :: lat(lsize) - real(r8) :: latMesh(lsize) - real(r8) :: lon(lsize) - real(r8) :: lonMesh(lsize) + real(r8), allocatable :: lat(:) + real(r8), allocatable :: latMesh(:) + real(r8), allocatable :: lon(:) + real(r8), allocatable :: lonMesh(:) integer :: ncols ! number of local columns integer :: start_ymd ! Start date (YYYYMMDD) integer :: start_tod ! Start time of day (sec) @@ -415,7 +415,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) call ESMF_LogWrite(subname//' called', ESMF_LOGMSG_INFO) end if - call shr_log_setLogUnit (iulog) + call shr_log_setLogUnit(iulog) !---------------------------------------------------------------------------- ! generate local mpi comm @@ -670,15 +670,39 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) call cam_set_mesh_for_single_column(scol_lon, scol_lat, model_mesh, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - allocate(dof(1), stat=ierr) - call check_allocate(ierr, subname, 'dof(1)', file=__FILE__, line=__LINE__) + allocate(dof(1), stat=ierr, errmsg=tempc1) + call check_allocate(ierr, subname, 'dof(1)', & + file=__FILE__, line=__LINE__, & + errmsg=tempc1) dof(1) = 1 else + ! Allocate lat/lon variables. Note that this must be done + ! after 'cam_init' or else the physics grid, and thus 'lsize', + ! won't be propertly set. + allocate(lat(lsize), stat=ierr, errmsg=tempc1) + call check_allocate(ierr, subname, 'lat(lsize)', & + file=__FILE__, line=__LINE__, & + errmsg=tempc1) + allocate(lon(lsize), stat=ierr, errmsg=tempc1) + call check_allocate(ierr, subname, 'lon(lsize)', & + file=__FILE__, line=__LINE__, & + errmsg=tempc1) + allocate(latMesh(lsize), stat=ierr, errmsg=tempc1) + call check_allocate(ierr, subname, 'latMesh(lsize)', & + file=__FILE__, line=__LINE__, & + errmsg=tempc1) + allocate(lonMesh(lsize), stat=ierr, errmsg=tempc1) + call check_allocate(ierr, subname, 'lonMesh(lsize)', & + file=__FILE__, line=__LINE__, & + errmsg=tempc1) + ! generate the dof - allocate(dof(lsize), stat=ierr) - call check_allocate(ierr, subname, 'dof(lsize)', file=__FILE__, line=__LINE__) + allocate(dof(lsize), stat=ierr, errmsg=tempc1) + call check_allocate(ierr, subname, 'dof(lsize)', & + file=__FILE__, line=__LINE__, & + errmsg=tempc1) do i = 1, lsize dof(i) = global_index_p(i) end do @@ -714,6 +738,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) file=__FILE__, line=__LINE__) call ESMF_MeshGet(model_mesh, ownedElemCoords=ownedElemCoords) if (ChkErr(rc,__LINE__,u_FILE_u)) return + write(*,*) 'DEBUG pre-loop -JN: ', lsize, size(ownedElemCoords), size(lonMesh), size(latMesh) do n = 1,lsize lonMesh(n) = ownedElemCoords(2*n-1) latMesh(n) = ownedElemCoords(2*n) From e2f0447bd4db81e4277ab3e7c308acc20ab586bf Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 13 Nov 2025 15:10:34 -0700 Subject: [PATCH 129/150] Add nitrogen deposition namelist and code updates needed for proper surface coupling. --- cime_config/cam_config.py | 9 +- src/control/runtime_opts.F90 | 8 +- src/cpl/nuopc/atm_comp_nuopc.F90 | 1 - src/cpl/nuopc/atm_import_export.F90 | 62 ++++---- src/cpl/nuopc/atm_stream_ndep.F90 | 143 +++++++++++------- .../namelist_definition_atm_stream_ndep.xml | 141 +++++++++++++++++ 6 files changed, 271 insertions(+), 93 deletions(-) create mode 100644 src/cpl/nuopc/namelist_definition_atm_stream_ndep.xml diff --git a/cime_config/cam_config.py b/cime_config/cam_config.py index fc0ddc9b..34415039 100644 --- a/cime_config/cam_config.py +++ b/cime_config/cam_config.py @@ -240,16 +240,21 @@ def __init__(self, case, case_log): # Save local (cime_config) directory path: cime_conf_path = os.path.dirname(os.path.abspath(__file__)) - # Save path to the "data" src direcotry: + # Save path to the "data" src directory: data_nml_path = os.path.join(cime_conf_path, os.pardir, "src", "data") + # Save path to the "cpl/nuopc" src directory: + cpl_nuopc_nml_path = os.path.join(cime_conf_path, os.pardir, "src", "cpl", + "nuopc") + # Create empty XML namelist definition files dictionary: self.__xml_nml_def_files = OrderedDict() #Add the default host model namelists: self._add_xml_nml_file(cime_conf_path, "namelist_definition_cam.xml") self._add_xml_nml_file(data_nml_path, "namelist_definition_physconst.xml") - self._add_xml_nml_file(data_nml_path, "namelist_definition_ref_pres.xml") + self._add_xml_nml_file(cpl_nuopc_nml_path, + "namelist_definition_atm_stream_ndep.xml") #---------------------------------------------------- # Set CAM start date (needed for namelist generation) diff --git a/src/control/runtime_opts.F90 b/src/control/runtime_opts.F90 index 7afe3147..0b59de0d 100644 --- a/src/control/runtime_opts.F90 +++ b/src/control/runtime_opts.F90 @@ -44,6 +44,7 @@ subroutine read_namelist(nlfilename, single_column, scmlat, scmlon) use radiation_namelist, only: radiation_readnl use dyn_comp, only: dyn_readnl + use atm_stream_ndep, only: stream_ndep_readnl !---------------------------Arguments----------------------------------- @@ -78,7 +79,6 @@ subroutine read_namelist(nlfilename, single_column, scmlat, scmlon) ! cam_read_ccpp_scheme_namelists call cam_logfile_readnl(nlfilename) -! call physics_grid_readnl(nlfilename) call physconst_readnl(nlfilename) call cam_initfiles_readnl(nlfilename) call cam_constituents_readnl(nlfilename) @@ -86,16 +86,12 @@ subroutine read_namelist(nlfilename, single_column, scmlat, scmlon) call phys_readnl(nlfilename) ! Should set phys_suite_name call vert_coord_readnl(nlfilename) call ref_pres_readnl(nlfilename) -! call phys_debug_readnl(nlfilename) -! call diag_readnl(nlfilename) -! call check_energy_readnl(nlfilename) call analytic_ic_readnl(nlfilename) call tropopause_climo_readnl(nlfilename) call radiation_readnl(nlfilename) -! call scam_readnl(nlfilename, single_column, scmlat, scmlon) -! call nudging_readnl(nlfilename) call gravity_wave_drag_ridge_read_readnl(nlfilename) call dyn_readnl(nlfilename) + call stream_ndep_readnl(nlfilename) ! Read the namelists for active physics schemes errflg = 0 diff --git a/src/cpl/nuopc/atm_comp_nuopc.F90 b/src/cpl/nuopc/atm_comp_nuopc.F90 index 245ee869..0b2842ee 100644 --- a/src/cpl/nuopc/atm_comp_nuopc.F90 +++ b/src/cpl/nuopc/atm_comp_nuopc.F90 @@ -738,7 +738,6 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc) file=__FILE__, line=__LINE__) call ESMF_MeshGet(model_mesh, ownedElemCoords=ownedElemCoords) if (ChkErr(rc,__LINE__,u_FILE_u)) return - write(*,*) 'DEBUG pre-loop -JN: ', lsize, size(ownedElemCoords), size(lonMesh), size(latMesh) do n = 1,lsize lonMesh(n) = ownedElemCoords(2*n-1) latMesh(n) = ownedElemCoords(2*n) diff --git a/src/cpl/nuopc/atm_import_export.F90 b/src/cpl/nuopc/atm_import_export.F90 index d1ebf171..15eab22e 100644 --- a/src/cpl/nuopc/atm_import_export.F90 +++ b/src/cpl/nuopc/atm_import_export.F90 @@ -709,7 +709,7 @@ subroutine import_fields( gcomp, cam_in, restart_init, rc) #endif #if 0 -! Commented out until water isotopes or carbon ccle fluxe are implemented in CAM-SIMA +! Commented out until water isotopes or carbon cycle fluxes are implemented in CAM-SIMA ! fields needed to calculate water isotopes to ocean evaporation processes call state_getfldptr(importState, 'So_ustar', fldptr=fldptr1d, exists=exists, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return @@ -853,14 +853,15 @@ subroutine export_fields( gcomp, model_mesh, model_clock, cam_out, rc) ! Copy from CAM-SIMA array data structure into state fldptr ! ----------------------------------------------------- - use ESMF , only : ESMF_Clock - use nuopc_shr_methods , only : chkerr - use srf_field_check , only : active_Faxa_nhx, active_Faxa_noy - use physics_types , only : cam_out_t - use time_manager , only : is_first_step, get_nstep - use physics_grid , only : columns_on_task - use atm_stream_ndep , only : stream_ndep_init, stream_ndep_interp - use atm_stream_ndep , only : stream_ndep_is_initialized + use ESMF , only: ESMF_Clock + use nuopc_shr_methods , only: chkerr + use srf_field_check , only: active_Faxa_nhx, active_Faxa_noy + use physics_types , only: cam_out_t + use time_manager , only: is_first_step, get_nstep + use physics_grid , only: columns_on_task + use atm_stream_ndep , only: stream_ndep_init, stream_ndep_interp + use atm_stream_ndep , only: stream_ndep_is_initialized + use atm_stream_ndep , only: ndep_stream_active !------------------------------- ! Pack the export state @@ -1033,29 +1034,36 @@ subroutine export_fields( gcomp, model_mesh, model_clock, cam_out, rc) end do end if - ! If ndep fields are not computed in cam and must be obtained from the ndep input stream call state_getfldptr(exportState, 'Faxa_ndep', fldptr2d=fldptr_ndep, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - if (.not. active_Faxa_nhx .and. .not. active_Faxa_noy) then - if (.not. stream_ndep_is_initialized) then - call stream_ndep_init(model_mesh, model_clock, rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - stream_ndep_is_initialized = .true. - end if + + fldptr_ndep(:,:) = 0._r8 + + ! The ndep_stream_nl namelist group is read in stream_ndep_init. This sets whether + ! or not the stream will be used. + if (.not. stream_ndep_is_initialized) then + call stream_ndep_init(model_mesh, model_clock, rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + stream_ndep_is_initialized = .true. + end if + + if (ndep_stream_active) then + + ! Nitrogen deposition fluxes are obtained + ! from the ndep input stream if input data + ! is available + + ! get ndep fluxes from the stream call stream_ndep_interp(cam_out, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return - ! NDEP read from forcing is expected to be in units of gN/m2/sec - but the mediator - ! expects units of kgN/m2/sec - scale_ndep = .001_r8 - else - ! If waccm computes ndep, then its in units of kgN/m2/s - and the mediator expects - ! units of kgN/m2/sec, so the following conversion needs to happen - scale_ndep = 1._r8 + + ! set field pointer to stream data + do i = 1, columns_on_task + fldptr_ndep(1,i) = cam_out%nhx_nitrogen_flx(i) * mod2med_areacor(i) + fldptr_ndep(2,i) = cam_out%noy_nitrogen_flx(i) * mod2med_areacor(i) + end do + end if - do i = 1, columns_on_task - fldptr_ndep(1,i) = cam_out%nhx_nitrogen_flx(i) * scale_ndep * mod2med_areacor(i) - fldptr_ndep(2,i) = cam_out%noy_nitrogen_flx(i) * scale_ndep * mod2med_areacor(i) - end do end subroutine export_fields diff --git a/src/cpl/nuopc/atm_stream_ndep.F90 b/src/cpl/nuopc/atm_stream_ndep.F90 index 2769f28a..10992a47 100644 --- a/src/cpl/nuopc/atm_stream_ndep.F90 +++ b/src/cpl/nuopc/atm_stream_ndep.F90 @@ -6,65 +6,63 @@ module atm_stream_ndep ! interpolation. !----------------------------------------------------------------------- ! - use ESMF , only : ESMF_Clock + use ESMF , only : ESMF_Clock, ESMF_Mesh + use ESMF , only : ESMF_SUCCESS, ESMF_LOGERR_PASSTHRU, ESMF_END_ABORT + use ESMF , only : ESMF_Finalize, ESMF_LogFoundError + use nuopc_shr_methods , only : chkerr use dshr_strdata_mod , only : shr_strdata_type - use shr_kind_mod , only : CS => shr_kind_cs + use shr_kind_mod , only : r8 => shr_kind_r8, CL => shr_kind_cl, CS => shr_kind_cs + use shr_log_mod , only : errMsg => shr_log_errMsg + use cam_logfile , only : iulog + use cam_abortutils , only : endrun implicit none private + public :: stream_ndep_readnl ! read runtime options public :: stream_ndep_init ! position datasets for dynamic ndep public :: stream_ndep_interp ! interpolates between two years of ndep file data private :: stream_ndep_check_units ! Check the units and make sure they can be used + ! The ndep stream is not needed for aquaplanet or simple model configurations. It + ! is disabled by setting the namelist variable stream_ndep_data_filename to 'UNSET' or empty string. + logical, public, protected :: ndep_stream_active = .false. + type(shr_strdata_type) :: sdat_ndep ! input data stream logical, public :: stream_ndep_is_initialized = .false. character(len=CS) :: stream_varlist_ndep(2) type(ESMF_Clock) :: model_clock - character(len=*), parameter :: sourcefile = & - __FILE__ + character(len=*), parameter :: sourcefile = __FILE__ + + character(len=CL) :: stream_ndep_data_filename + character(len=CL) :: stream_ndep_mesh_filename + integer :: stream_ndep_year_first ! first year in stream to use + integer :: stream_ndep_year_last ! last year in stream to use + integer :: stream_ndep_year_align ! align stream_year_firstndep with !============================================================================== contains !============================================================================== - subroutine stream_ndep_init(model_mesh, model_clock, rc) - ! - ! Initialize data stream information. + subroutine stream_ndep_readnl(nlfile) ! Uses: - use ESMF , only: ESMF_Mesh - use ESMF , only: ESMF_SUCCESS, ESMF_LOGERR_PASSTHRU, ESMF_END_ABORT - use ESMF , only: ESMF_Finalize, ESMF_LogFoundError - use cam_instance , only: inst_suffix - use shr_nl_mod , only: shr_nl_find_group_name - use shr_kind_mod , only: CL => shr_kind_cl - use shr_kind_mod , only: r8 => shr_kind_r8 - use shr_log_mod , only: errMsg => shr_log_errMsg - use dshr_strdata_mod , only: shr_strdata_init_from_inline - use mpi , only: mpi_character, mpi_integer - use spmd_utils , only: mpicom, masterproc, iam - use cam_logfile , only: iulog - use cam_abortutils , only: endrun + use shr_nl_mod, only: shr_nl_find_group_name + use runtime_obj, only: unset_str + use spmd_utils, only: mpicom, masterproc + use mpi, only: mpi_character, mpi_integer ! input/output variables - type(ESMF_CLock), intent(in) :: model_clock - type(ESMF_Mesh) , intent(in) :: model_mesh - integer , intent(out) :: rc + character(len=*), intent(in) :: nlfile ! local variables integer :: nu_nml ! unit for namelist file integer :: nml_error ! namelist i/o error flag - character(len=CL) :: stream_ndep_data_filename - character(len=CL) :: stream_ndep_mesh_filename - character(len=CL) :: filein ! atm namelist file - integer :: stream_ndep_year_first ! first year in stream to use - integer :: stream_ndep_year_last ! last year in stream to use - integer :: stream_ndep_year_align ! align stream_year_firstndep with + character(len=CL) :: nml_errmsg ! namelist i/o error message integer :: ierr - character(*), parameter :: subName = "('stream_ndep_init')" + character(*), parameter :: subName = "('stream_ndep_readnl')" !----------------------------------------------------------------------- namelist /ndep_stream_nl/ & @@ -74,8 +72,6 @@ subroutine stream_ndep_init(model_mesh, model_clock, rc) stream_ndep_year_last, & stream_ndep_year_align - rc = ESMF_SUCCESS - ! Default values for namelist stream_ndep_data_filename = ' ' stream_ndep_mesh_filename = ' ' @@ -88,19 +84,18 @@ subroutine stream_ndep_init(model_mesh, model_clock, rc) ! Read ndep_stream namelist if (masterproc) then - filein = "atm_in" // trim(inst_suffix) - open( newunit=nu_nml, file=trim(filein), status='old', iostat=nml_error ) + open(newunit=nu_nml, file=trim(nlfile), status='old', iostat=nml_error, iomsg=nml_errmsg) if (nml_error /= 0) then - call endrun(subName//': ERROR opening '//trim(filein)//errMsg(sourcefile, __LINE__)) + call endrun(subName//': ERROR opening '//trim(nlfile)//' : '//& + errMsg(sourcefile, __LINE__)//' : '//trim(nml_errmsg)) end if call shr_nl_find_group_name(nu_nml, 'ndep_stream_nl', status=nml_error) if (nml_error == 0) then - read(nu_nml, nml=ndep_stream_nl, iostat=nml_error) + read(nu_nml, nml=ndep_stream_nl, iostat=nml_error, iomsg=nml_errmsg) if (nml_error /= 0) then - call endrun(' ERROR reading ndep_stream_nl namelist'//errMsg(sourcefile, __LINE__)) + call endrun(' ERROR reading ndep_stream_nl namelist: '//& + errMsg(sourcefile, __LINE__)//' : '//trim(nml_errmsg)) end if - else - call endrun(' ERROR finding ndep_stream_nl namelist'//errMsg(sourcefile, __LINE__)) end if close(nu_nml) endif @@ -115,6 +110,19 @@ subroutine stream_ndep_init(model_mesh, model_clock, rc) call mpi_bcast(stream_ndep_year_align, 1, mpi_integer, 0, mpicom, ierr) if (ierr /= 0) call endrun(trim(subname)//": FATAL: mpi_bcast: stream_ndep_year_align") + ndep_stream_active = (len_trim(stream_ndep_data_filename)>0) .and. & + (stream_ndep_data_filename /= unset_str) + + ! Check whether the stream is being used. + if (.not.ndep_stream_active) then + if (masterproc) then + write(iulog,'(a)') ' ' + write(iulog,'(a)') 'NDEP STREAM IS NOT USED.' + write(iulog,'(a)') ' ' + endif + return + endif + if (masterproc) then write(iulog,'(a)' ) ' ' write(iulog,'(a,i8)') 'stream ndep settings:' @@ -127,6 +135,28 @@ subroutine stream_ndep_init(model_mesh, model_clock, rc) write(iulog,'(a)' ) ' ' endif + end subroutine stream_ndep_readnl + + subroutine stream_ndep_init(model_mesh, model_clock, rc) + + ! Uses: + use dshr_strdata_mod, only: shr_strdata_init_from_inline + use spmd_utils, only: iam + + ! input/output variables + type(ESMF_CLock), intent(in) :: model_clock + type(ESMF_Mesh) , intent(in) :: model_mesh + integer , intent(out) :: rc + + ! local variables + character(*), parameter :: subName = "('stream_ndep_init')" + + rc = ESMF_SUCCESS + if (.not.ndep_stream_active) then + return + end if + ! + ! Initialize data stream information. ! Read in units call stream_ndep_check_units(stream_ndep_data_filename) @@ -165,12 +195,10 @@ subroutine stream_ndep_check_units( stream_fldFileName_ndep) ! Check that units are correct on the file and if need any conversion !-------------------------------------------------------- - use cam_pio_utils , only : cam_pio_createfile, cam_pio_openfile, cam_pio_closefile, pio_subsystem - use pio , only : file_desc_t, io_desc_t, var_desc_t, pio_double, pio_def_dim - use pio , only : pio_bcast_error, pio_seterrorhandling, pio_inq_varid, pio_get_att - use pio , only : PIO_NOERR, PIO_NOWRITE - use shr_log_mod , only : errMsg => shr_log_errMsg - use cam_abortutils , only : endrun + use cam_pio_utils , only : cam_pio_createfile, cam_pio_openfile, cam_pio_closefile, pio_subsystem + use pio , only : file_desc_t, io_desc_t, var_desc_t, pio_double, pio_def_dim + use pio , only : pio_bcast_error, pio_seterrorhandling, pio_inq_varid, pio_get_att + use pio , only : PIO_NOERR, PIO_NOWRITE ! Arguments character(len=*), intent(in) :: stream_fldFileName_ndep ! ndep filename @@ -206,22 +234,18 @@ end subroutine stream_ndep_check_units !================================================================ subroutine stream_ndep_interp(cam_out, rc) - use ESMF , only : ESMF_LOGERR_PASSTHRU, ESMF_END_ABORT - use ESMF , only : ESMF_Finalize, ESMF_LogFoundError - use dshr_methods_mod , only : dshr_fldbun_getfldptr - use dshr_strdata_mod , only : shr_strdata_advance - use shr_kind_mod , only : r8 => shr_kind_r8 - use physics_types , only : cam_out_t - use time_manager , only : get_curr_date - use physics_grid , only : columns_on_task - use cam_logfile , only : iulog + use dshr_methods_mod , only: dshr_fldbun_getfldptr + use dshr_strdata_mod , only: shr_strdata_advance + use physics_types , only: cam_out_t + use physics_grid , only: columns_on_task + use time_manager , only: get_curr_date ! input/output variables type(cam_out_t) , intent(inout) :: cam_out integer , intent(out) :: rc ! local variables - integer :: i + integer :: i ! column loop variable integer :: year ! year (0, ...) for nstep+1 integer :: mon ! month (1, ..., 12) for nstep+1 integer :: day ! day of month (1, ..., 31) for nstep+1 @@ -229,6 +253,11 @@ subroutine stream_ndep_interp(cam_out, rc) integer :: mcdate ! Current model date (yyyymmdd) real(r8), pointer :: dataptr1d_nhx(:) real(r8), pointer :: dataptr1d_noy(:) + + ! NDEP read from forcing is expected to be in units of gN/m2/sec - but the mediator + ! expects units of kgN/m2/sec + real(r8), parameter :: scale_ndep = .001_r8 + !----------------------------------------------------------------------- ! Advance sdat stream @@ -250,8 +279,8 @@ subroutine stream_ndep_interp(cam_out, rc) end if do i = 1, columns_on_task - cam_out%nhx_nitrogen_flx(i) = dataptr1d_nhx(i) - cam_out%noy_nitrogen_flx(i) = dataptr1d_noy(i) + cam_out%nhx_nitrogen_flx(i) = dataptr1d_nhx(i) * scale_ndep + cam_out%noy_nitrogen_flx(i) = dataptr1d_noy(i) * scale_ndep end do end subroutine stream_ndep_interp diff --git a/src/cpl/nuopc/namelist_definition_atm_stream_ndep.xml b/src/cpl/nuopc/namelist_definition_atm_stream_ndep.xml new file mode 100644 index 00000000..5cd32033 --- /dev/null +++ b/src/cpl/nuopc/namelist_definition_atm_stream_ndep.xml @@ -0,0 +1,141 @@ + + + + + + + + + + char*256 + Nitrogen deposition + ndep_stream_nl + + Nitrogen deposition stream data filename. + If filename is "UNSET", then no nitrogen + deposition will be done. + + + UNSET + + + + + char*256 + Nitrogen deposition + ndep_stream_nl + + Grid mesh file corresponding to stream_ndep_data_filename. + + + UNSET + + + + + integer + Nitrogen deposition + ndep_stream_nl + + First year to use in nitrogen deposition stream data. + + + 1 + + + + + integer + Nitrogen deposition + ndep_stream_nl + + Last year to use in nitrogen deposition stream data. + + + 1 + + + + + integer + Nitrogen deposition + ndep_stream_nl + + Model year to align with stream_ndep_year_first. + + + 1 + + + + From 0bb3cf4395cd09bbd6a2df417f3713dbee86ab0f Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 13 Nov 2025 15:23:14 -0700 Subject: [PATCH 130/150] Uncomment 'nextsw_cday' calls in the NUOPC cap. --- src/cpl/nuopc/atm_comp_nuopc.F90 | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/cpl/nuopc/atm_comp_nuopc.F90 b/src/cpl/nuopc/atm_comp_nuopc.F90 index 0b2842ee..d67b08c4 100644 --- a/src/cpl/nuopc/atm_comp_nuopc.F90 +++ b/src/cpl/nuopc/atm_comp_nuopc.F90 @@ -44,7 +44,6 @@ module atm_comp_nuopc use cam_comp , only : cam_init, cam_run1, cam_run2, cam_run3, cam_run4, cam_final use cam_comp , only : cam_timestep_init, cam_timestep_final use physics_types , only : cam_out, cam_in -! use radiation , only : nextsw_cday !uncomment once radiation has been CCPP-ized -JN use cam_logfile , only : cam_set_log_unit, iulog use cam_abortutils , only : check_allocate use spmd_utils , only : spmd_init, masterproc, iam @@ -820,6 +819,8 @@ end subroutine InitializeRealize !=============================================================================== subroutine DataInitialize(gcomp, rc) + use physics_types, only: nextsw_cday + type(ESMF_GridComp) :: gcomp integer, intent(out) :: rc @@ -952,13 +953,10 @@ subroutine DataInitialize(gcomp, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return end if -!Remove once radiation (nextsw_cday) has been enabled in CAM-SIMA -JN. -#if 0 ! Compute time of next radiation computation call State_SetScalar(nextsw_cday, flds_scalar_index_nextsw_cday, exportState, & flds_scalar_name, flds_scalar_num, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return -#endif ! diagnostics if (dbug_flag > 1) then @@ -1026,7 +1024,8 @@ end subroutine DataInitialize !=============================================================================== subroutine ModelAdvance(gcomp, rc) - use ESMF, only : ESMF_GridCompGet, esmf_vmget, esmf_vm + use ESMF, only: ESMF_GridCompGet, esmf_vmget, esmf_vm + use physics_types, only: nextsw_cday ! Run CAM ! Input/output variables @@ -1213,14 +1212,11 @@ subroutine ModelAdvance(gcomp, rc) call t_stopf ('CAM_export') ! Set the coupling scalars -!Remove once radiation (nextsw_cday) has been enabled in CAM-SIMA -JN. -#if 0 ! Return time of next radiation calculation - albedos will need to be ! calculated by each surface model at this time call State_SetScalar(nextsw_cday, flds_scalar_index_nextsw_cday, exportState, & flds_scalar_name, flds_scalar_num, rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return -#endif ! diagnostics if (dbug_flag > 1) then From b23d19f1e709ca1a1443ae180ff4b4fbdd594085 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 14 Nov 2025 10:15:19 -0700 Subject: [PATCH 131/150] Skip consitutuents input read if variable not found on file and 'error_on_not_found' is False. --- src/physics/utils/physics_data.F90 | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/physics/utils/physics_data.F90 b/src/physics/utils/physics_data.F90 index 7cf1f8ef..d0adeedf 100644 --- a/src/physics/utils/physics_data.F90 +++ b/src/physics/utils/physics_data.F90 @@ -444,8 +444,18 @@ subroutine read_constituent_dimensioned_field_2d(const_props, file, std_name, ba end do const_idx_loop end do base_idx_loop - if(.not. var_found .and. error_on_not_found_local) then - call endrun(subname//'Required constituent-dimensioned variables not found: No match for ' // trim(std_name)) + if(.not. var_found) then + if(error_on_not_found_local) then + !End model run with appropriate error message: + call endrun(subname // 'Required constituent-dimensioned variables not found: No match for ' // trim(std_name)) + else + !Write message to log file, then exit subroutine: + if (masterproc) then + write(iulog, *) subname // 'Required constituent-dimensioned variables not found: No match for ' // trim(std_name) + call shr_sys_flush(iulog) + end if + return !Nothing more to do here + end if end if ! Once base_idx is identified, use it in the actual constituent loop: From d7bc6f0e8b3635ad1d38b19f15cb2d18fc22c01e Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 17 Nov 2025 15:18:24 -0700 Subject: [PATCH 132/150] Add constituents info to atm log file, and fix bad MPAS dyn_coupling constituent indexing. --- src/dynamics/mpas/dyn_coupling_impl.F90 | 10 ++-- src/physics/utils/cam_constituents.F90 | 67 +++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/src/dynamics/mpas/dyn_coupling_impl.F90 b/src/dynamics/mpas/dyn_coupling_impl.F90 index 68eebfb9..b530ff55 100644 --- a/src/dynamics/mpas/dyn_coupling_impl.F90 +++ b/src/dynamics/mpas/dyn_coupling_impl.F90 @@ -492,7 +492,7 @@ end subroutine set_physics_state_column subroutine set_physics_state_external() ! Module(s) from CAM-SIMA. use cam_abortutils, only: check_allocate, endrun - use cam_constituents, only: const_qmin, num_advected + use cam_constituents, only: const_qmin, num_constituents use cam_thermo, only: cam_thermo_dry_air_update, cam_thermo_water_update use cam_thermo_formula, only: energy_formula_dycore_mpas use dyn_comp, only: mpas_dynamical_core @@ -525,12 +525,12 @@ subroutine set_physics_state_external() nullify(constituents) nullify(constituent_properties) - allocate(minimum_constituents(num_advected), errmsg=cerr, stat=ierr) + allocate(minimum_constituents(num_constituents), errmsg=cerr, stat=ierr) call check_allocate(ierr, subname, & - 'minimum_constituents(num_advected)', & + 'minimum_constituents(num_constituents)', & file='dyn_coupling', line=__LINE__, errmsg=trim(adjustl(cerr))) - do i = 1, num_advected + do i = 1, num_constituents minimum_constituents(i) = const_qmin(i) end do @@ -579,7 +579,7 @@ subroutine set_physics_state_external() ! Set `zi` (i.e., geopotential height at layer interfaces) and `zm` (i.e., geopotential height at layer midpoints). ! Note that `rairv` and `zvirv` are updated externally by `cam_thermo_dry_air_update`. call geopotential_temp_run( & - pver, lagrangian_vertical, pver, 1, pverp, 1, num_advected, & + pver, lagrangian_vertical, pver, 1, pverp, 1, num_constituents, & phys_state % lnpint, phys_state % pint, phys_state % pmid, phys_state % pdel, phys_state % rpdel, phys_state % t, & constituents(:, :, mpas_dynamical_core % map_constituent_index(index_qv)), constituents, & constituent_properties, rairv, constant_g, zvirv, phys_state % zi, phys_state % zm, ncells_solve, ierr, cerr) diff --git a/src/physics/utils/cam_constituents.F90 b/src/physics/utils/cam_constituents.F90 index 19329153..8202981d 100644 --- a/src/physics/utils/cam_constituents.F90 +++ b/src/physics/utils/cam_constituents.F90 @@ -14,6 +14,7 @@ module cam_constituents public :: const_longname public :: const_molec_weight public :: const_get_index + public :: const_is_advected public :: const_is_dry public :: const_is_moist public :: const_is_wet @@ -37,11 +38,18 @@ module cam_constituents !! \htmlinclude cam_constituents.html integer, public, protected :: num_advected = 0 + integer, public, protected :: num_constituents = 0 + !! Note: There are no _name interfaces in function interfaces below !! because use of this sort of interface is often for optional !! constituents and there is no way to indicate a missing !! constituent in these functions (e.g., a logical). + interface const_is_advected + module procedure const_is_advected_obj + module procedure const_is_advected_index + end interface const_is_advected + interface const_is_dry module procedure const_is_dry_obj module procedure const_is_dry_index @@ -156,20 +164,41 @@ end subroutine cam_constituents_readnl subroutine cam_constituents_init(cnst_prop_ptr, num_advect) use cam_abortutils, only: endrun + use cam_logfile, only: iulog, debug_output + use cam_logfile, only: DEBUGOUT_VERBOSE ! Initialize module constituent variables type(ccpp_constituent_prop_ptr_t), pointer :: cnst_prop_ptr(:) integer, intent(in) :: num_advect + !For log output: + integer :: cnst_idx + if (initialized) then call endrun("cam_constituents_init: already initialized", & file=__FILE__, line=__LINE__) end if const_props => cnst_prop_ptr num_advected = num_advect + num_constituents = size(const_props) initialized = .true. + !If log level is verbose, then print out + !the names/order of all registered constituents: + if (debug_output >= DEBUGOUT_VERBOSE) then + + write(iulog,*) 'LIST OF REGISTERED CONSTITUENTS:' + write(iulog,*) '********************************' + write(iulog,*) ' Constituent index : Standard name : Advected (T or F)' + do cnst_idx = 1, num_constituents + write(iulog,'(I0,3A, L)') cnst_idx, ' : ', trim(const_name(cnst_idx)), ' : ', & + const_is_advected(cnst_idx) + end do + write(iulog,*) '********************************' + + end if + end subroutine cam_constituents_init !####################################################################### @@ -364,6 +393,44 @@ end subroutine const_get_index !####################################################################### + logical function const_is_advected_obj(const_obj) + use cam_abortutils, only: endrun + use string_utils, only: to_str + + ! Return .true. if the constituent object, , is advected + ! Dummy argument + type(ccpp_constituent_prop_ptr_t), intent(in) :: const_obj + ! Local variables + integer :: err_code + character(len=256) :: err_msg + character(len=*), parameter :: subname = 'const_is_advected_obj: ' + + call const_obj%is_advected(const_is_advected_obj, err_code, err_msg) + if (err_code /= 0) then + call endrun(subname//"Error "//to_str(err_code)//": "// & + trim(err_msg), file=__FILE__, line=__LINE__) + end if + + end function const_is_advected_obj + + !####################################################################### + + logical function const_is_advected_index(const_ind) + + ! Return .true. if the constituent at is advected + ! Dummy argument + integer, intent(in) :: const_ind + ! Local variable + character(len=*), parameter :: subname = 'const_is_advected_index: ' + + if (check_index_bounds(const_ind, subname)) then + const_is_advected_index = const_is_advected(const_props(const_ind)) + end if + + end function const_is_advected_index + + !####################################################################### + logical function const_is_dry_obj(const_obj) use cam_abortutils, only: endrun use string_utils, only: to_str From f25da303360a09a22226588ee201702b36f0c9db Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 18 Nov 2025 13:07:05 -0700 Subject: [PATCH 133/150] Initial attempt at only using advected constituents in MPAS dycore. --- src/dynamics/mpas/dyn_comp_impl.F90 | 37 ++++++++++++++++++----- src/dynamics/mpas/dyn_coupling_impl.F90 | 39 +++++++++++++++++++------ 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/src/dynamics/mpas/dyn_comp_impl.F90 b/src/dynamics/mpas/dyn_comp_impl.F90 index 600e9f5e..9489e794 100644 --- a/src/dynamics/mpas/dyn_comp_impl.F90 +++ b/src/dynamics/mpas/dyn_comp_impl.F90 @@ -128,7 +128,9 @@ end subroutine dyn_readnl module subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) ! Module(s) from CAM-SIMA. use cam_abortutils, only: check_allocate - use cam_constituents, only: const_name, const_is_water_species, num_advected, readtrace + use cam_constituents, only: const_name, const_is_water_species + use cam_constituents, only: num_constituents, num_advected + use cam_constituents, only: const_is_advected, readtrace use cam_control_mod, only: initial_run use cam_initfiles, only: initial_file_get_id, topo_file_get_id use cam_logfile, only: debugout_debug, debugout_info @@ -174,9 +176,11 @@ module subroutine dyn_init(cam_runtime_opts, dyn_in, dyn_out) call check_allocate(ierr, subname, 'is_water_species(num_advected)', & file='dyn_comp', line=__LINE__, errmsg=trim(adjustl(cerr))) - do i = 1, num_advected - constituent_name(i) = const_name(i) - is_water_species(i) = const_is_water_species(i) + do i = 1, num_constituents + if (const_is_advected(i)) then !Skip non-advected consitutents + constituent_name(i) = const_name(i) + is_water_species(i) = const_is_water_species(i) + end if end do call dyn_debug_print(debugout_info, 'Defining MPAS scalars and scalar tendencies') @@ -614,6 +618,8 @@ subroutine set_mpas_state_scalars() ! Module(s) from CAM-SIMA. use cam_abortutils, only: check_allocate use cam_constituents, only: num_advected + use cam_constituents, only: num_constituents + use cam_constituents, only: const_is_advected use cam_logfile, only: debugout_verbose use dyn_grid, only: ncells_solve use dyn_procedures, only: qv_of_sh, reverse @@ -652,7 +658,19 @@ subroutine set_mpas_state_scalars() call mpas_dynamical_core % get_variable_pointer(scalars, 'state', 'scalars', time_level=1) buffer_3d_real(:, :, :) = 0.0_kind_r8 - constituent_index(:) = [(i, i = 1, num_advected)] + + ! Extract indices for all advected constituents. + j = 1 + do i = 1, num_constituents + if (const_is_advected(i)) then + constituent_index(j) = i + if (j == num_advected) then + exit !Break out of loop if no more advected constituents + else + j = j + 1 + end if + end if + end do call dyn_set_inic_col(vc_height, lat_rad, lon_rad, global_grid_index, zint=z_int, q=buffer_3d_real, & m_cnst=constituent_index) @@ -913,7 +931,8 @@ end subroutine set_analytic_initial_condition !> (KCW, 2024-05-23) subroutine mark_variables_as_initialized() ! Module(s) from CAM-SIMA. - use cam_constituents, only: const_name, num_advected + use cam_constituents, only: num_constituents, num_advected + use cam_constituents, only: const_name, const_is_advected use cam_logfile, only: debugout_debug ! Module(s) from CCPP. use phys_vars_init_check, only: mark_as_initialized @@ -960,8 +979,10 @@ subroutine mark_variables_as_initialized() call mark_as_initialized('tendency_of_northward_wind_due_to_model_physics') ! CCPP standard names of constituents. - do i = 1, num_advected - call mark_as_initialized(trim(adjustl(const_name(i)))) + do i = 1, num_constituents + if (const_is_advected(i)) then + call mark_as_initialized(trim(adjustl(const_name(i)))) + end if end do ! The variables below are not managed by dynamics interface. They are used by external CCPP physics schemes. diff --git a/src/dynamics/mpas/dyn_coupling_impl.F90 b/src/dynamics/mpas/dyn_coupling_impl.F90 index b530ff55..60438c45 100644 --- a/src/dynamics/mpas/dyn_coupling_impl.F90 +++ b/src/dynamics/mpas/dyn_coupling_impl.F90 @@ -24,7 +24,8 @@ module subroutine dyn_exchange_constituent_states(direction, exchange, conversion) ! Module(s) from CAM-SIMA. use cam_abortutils, only: check_allocate, endrun - use cam_constituents, only: const_is_dry, const_is_water_species, num_advected + use cam_constituents, only: num_constituents, num_advected + use cam_constituents, only: const_is_dry, const_is_water_species, const_is_advected use cam_logfile, only: debugout_debug, debugout_info use dyn_comp, only: dyn_debug_print, kind_dyn_mpas, mpas_dynamical_core use dyn_grid, only: ncells_solve @@ -33,6 +34,7 @@ module subroutine dyn_exchange_constituent_states(direction, exchange, conversio use vert_coord, only: pver ! Module(s) from CCPP. use cam_ccpp_cap, only: cam_constituents_array + use cam_ccpp_cap, only: cam_advected_constituents_array use ccpp_kinds, only: kind_phys ! Module(s) from CESM Share. use shr_kind_mod, only: kind_r8 => shr_kind_r8, & @@ -89,11 +91,20 @@ module subroutine dyn_exchange_constituent_states(direction, exchange, conversio 'is_water_species(num_advected)', & file='dyn_coupling', line=__LINE__, errmsg=trim(adjustl(cerr))) - do j = 1, num_advected + + j = 1 + do i = 1, num_constituents ! All constituent mixing ratios in MPAS are dry. ! Therefore, conversion in between is needed for any constituent mixing ratios that are not dry in CAM-SIMA. - is_conversion_needed(j) = .not. const_is_dry(j) - is_water_species(j) = const_is_water_species(j) + if (const_is_advected(i)) then + is_conversion_needed(j) = .not. const_is_dry(i) + is_water_species(j) = const_is_water_species(i) + if (i == num_advected) then + exit !All advected constituents accounted for, so exit loop. + else + j = j + 1 + end if + end if end do allocate(is_water_species_index(count(is_water_species)), errmsg=cerr, stat=ierr) @@ -106,7 +117,8 @@ module subroutine dyn_exchange_constituent_states(direction, exchange, conversio 'sigma_all_q(pver)', & file='dyn_coupling', line=__LINE__, errmsg=trim(adjustl(cerr))) - constituents => cam_constituents_array() + !constituents => cam_constituents_array() + constituents => cam_advected_constituents_array() if (.not. associated(constituents)) then call endrun('Failed to find variable "constituents"', subname, __LINE__) @@ -253,7 +265,8 @@ module subroutine dynamics_to_physics_coupling() subroutine init_shared_variables() ! Module(s) from CAM-SIMA. use cam_abortutils, only: check_allocate - use cam_constituents, only: const_is_water_species, num_advected + use cam_constituents, only: num_constituents, num_advected + use cam_constituents, only: const_is_water_species, const_is_advected use dyn_comp, only: mpas_dynamical_core use vert_coord, only: pver, pverp ! Module(s) from CESM Share. @@ -261,7 +274,7 @@ subroutine init_shared_variables() character(*), parameter :: subname = 'dyn_coupling::dynamics_to_physics_coupling::init_shared_variables' character(len_cx) :: cerr - integer :: i + integer :: i, j integer :: ierr logical, allocatable :: is_water_species(:) @@ -281,8 +294,16 @@ subroutine init_shared_variables() 'is_water_species(num_advected)', & file='dyn_coupling', line=__LINE__, errmsg=trim(adjustl(cerr))) - do i = 1, num_advected - is_water_species(i) = const_is_water_species(i) + j = 1 + do i = 1, num_constituents + if (const_is_advected(i)) then + is_water_species(j) = const_is_water_species(i) + if (j == num_advected) then + exit !All advected species accounted for, so exit loop + else + j = j + 1 + end if + end if end do allocate(is_water_species_index(count(is_water_species)), errmsg=cerr, stat=ierr) From c260411886c01b5f17a40b3ed0c5f77113a49981 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 20 Nov 2025 15:40:36 -0700 Subject: [PATCH 134/150] Only print constituents to log if root proc, and update atmos_phys external to fix bugs. --- .gitmodules | 2 +- src/physics/ncar_ccpp | 2 +- src/physics/utils/cam_constituents.F90 | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 80b7b318..e9bcaafa 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/nusbaume/atmospheric_physics - fxtag = 9011861b0534819fa798ae5eb745c0573c5dad22 + fxtag = 159ec014cd1dee148f62e34efeddcfe510b53b1d fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 9011861b..159ec014 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 9011861b0534819fa798ae5eb745c0573c5dad22 +Subproject commit 159ec014cd1dee148f62e34efeddcfe510b53b1d diff --git a/src/physics/utils/cam_constituents.F90 b/src/physics/utils/cam_constituents.F90 index 8202981d..eabddffa 100644 --- a/src/physics/utils/cam_constituents.F90 +++ b/src/physics/utils/cam_constituents.F90 @@ -164,6 +164,7 @@ end subroutine cam_constituents_readnl subroutine cam_constituents_init(cnst_prop_ptr, num_advect) use cam_abortutils, only: endrun + use spmd_utils, only: masterproc use cam_logfile, only: iulog, debug_output use cam_logfile, only: DEBUGOUT_VERBOSE @@ -186,7 +187,7 @@ subroutine cam_constituents_init(cnst_prop_ptr, num_advect) !If log level is verbose, then print out !the names/order of all registered constituents: - if (debug_output >= DEBUGOUT_VERBOSE) then + if ((debug_output >= DEBUGOUT_VERBOSE) .and. masterproc) then write(iulog,*) 'LIST OF REGISTERED CONSTITUENTS:' write(iulog,*) '********************************' From f0c69b2ba0b987d851aae2382a4519118069c3b9 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 24 Nov 2025 15:24:13 -0700 Subject: [PATCH 135/150] Update atmos_phys submodule and fix bugs in order to properly set surface coupling fields. --- .gitmodules | 2 +- src/cpl/nuopc/atm_import_export.F90 | 10 ++++++++-- src/data/generate_registry_data.py | 2 +- src/physics/ncar_ccpp | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index e9bcaafa..70ca5d89 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/nusbaume/atmospheric_physics - fxtag = 159ec014cd1dee148f62e34efeddcfe510b53b1d + fxtag = 7d5dd0125802e38b2e5b119a36e206da95b50e02 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/cpl/nuopc/atm_import_export.F90 b/src/cpl/nuopc/atm_import_export.F90 index 15eab22e..594781ec 100644 --- a/src/cpl/nuopc/atm_import_export.F90 +++ b/src/cpl/nuopc/atm_import_export.F90 @@ -862,6 +862,7 @@ subroutine export_fields( gcomp, model_mesh, model_clock, cam_out, rc) use atm_stream_ndep , only: stream_ndep_init, stream_ndep_interp use atm_stream_ndep , only: stream_ndep_is_initialized use atm_stream_ndep , only: ndep_stream_active + use cam_constituents , only: const_get_index !------------------------------- ! Pack the export state @@ -876,7 +877,8 @@ subroutine export_fields( gcomp, model_mesh, model_clock, cam_out, rc) ! local variables type(ESMF_State) :: exportState - integer :: i ! index variable + integer :: i ! Loop index variable + integer :: ix_qv ! Constituents index for water vapor logical :: exists real(r8) :: scale_ndep ! 2d pointers @@ -902,6 +904,10 @@ subroutine export_fields( gcomp, model_mesh, model_clock, cam_out, rc) rc = ESMF_SUCCESS + ! Determine water vapor constituent index + call const_get_index('water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water', & + ix_qv) + ! Get export state call NUOPC_ModelGet(gcomp, exportState=exportState, rc=rc) if (ChkErr(rc,__LINE__,u_FILE_u)) return @@ -934,7 +940,7 @@ subroutine export_fields( gcomp, model_mesh, model_clock, cam_out, rc) fldptr_vbot(i) = cam_out%vbot(i) fldptr_pbot(i) = cam_out%pbot(i) fldptr_tbot(i) = cam_out%tbot(i) - fldptr_shum(i) = cam_out%qbot(i,1) + fldptr_shum(i) = cam_out%qbot(i, ix_qv) fldptr_dens(i) = cam_out%rho(i) fldptr_ptem(i) = cam_out%thbot(i) fldptr_pslv(i) = cam_out%psl(i) diff --git a/src/data/generate_registry_data.py b/src/data/generate_registry_data.py index 30c09f4a..2ddcf152 100755 --- a/src/data/generate_registry_data.py +++ b/src/data/generate_registry_data.py @@ -1404,7 +1404,7 @@ def write_allocate_routine(self, outfile, physconst_vars, var_module_dict): #supports a separation between total and advected constituents #then this section of code will likely need to be modified: if ('number_of_ccpp_constituents' in self.__var_dict.known_dimensions): - outfile.write("use cam_constituents, only: number_of_ccpp_constituents=>num_advected", 2) + outfile.write("use cam_constituents, only: number_of_ccpp_constituents=>num_constituents", 2) outfile.blank_line() # Dummy arguments diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 159ec014..7d5dd012 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 159ec014cd1dee148f62e34efeddcfe510b53b1d +Subproject commit 7d5dd0125802e38b2e5b119a36e206da95b50e02 From 701ad8f65c668c4bf6420333bdef3d40d470e3f1 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 26 Nov 2025 10:13:15 -0700 Subject: [PATCH 136/150] Fix namelist bug, update PEs file, and modify physics_read for better logging and running frequency. --- cime_config/config_pes.xml | 2459 ++++++++++------------- cime_config/namelist_definition_cam.xml | 3 +- src/physics/utils/phys_comp.F90 | 26 +- src/physics/utils/physics_data.F90 | 8 +- 4 files changed, 1135 insertions(+), 1361 deletions(-) diff --git a/cime_config/config_pes.xml b/cime_config/config_pes.xml index 6d7bafe5..7c8c2b95 100644 --- a/cime_config/config_pes.xml +++ b/cime_config/config_pes.xml @@ -1,79 +1,45 @@ - - - none - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - none - -1 - -1 - -1 - -1 - -1 - -1 - -1 - -1 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + -1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 - - + + none @@ -109,150 +75,150 @@ - + - none - - -4 - -4 - -4 - -4 - -4 - -4 - -4 - -4 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - + none + + 24 + 24 + 24 + 24 + 24 + 24 + 24 + 24 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + - + none - -4 - -4 - -4 - -4 - -4 - -4 - -4 - -4 + -4 + -4 + -4 + -4 + -4 + -4 + -4 + -4 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 - + none - -40 - -40 - -40 - -40 - -40 - -40 - -40 - -40 + -4 + -4 + -4 + -4 + -4 + -4 + -4 + -4 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 - + none - -40 - -40 - -40 - -40 - -40 - -40 - -40 - -40 + -40 + -40 + -40 + -40 + -40 + -40 + -40 + -40 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 @@ -262,361 +228,235 @@ none - 2048 - 2048 - 2048 - 2048 - 2048 - 2048 - 2048 - 2048 + 2048 + 2048 + 2048 + 2048 + 2048 + 2048 + 2048 + 2048 - 8 - 8 - 8 - 8 - 8 - 8 - 8 - 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 - - + + - none - - -8 - -8 - -8 - -8 - -8 - -8 - -8 - -8 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - none - -16 - -16 - -16 - -16 - -16 - -16 - -16 - -16 + -4 + -4 + -4 + -4 + -4 + -4 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 + 1 + 1 + 1 + 1 + 1 + 1 - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - 16 - 32 + + none - 630 - 630 - 630 - 630 - 630 - 630 - 630 - 630 + -4 + -4 + -4 + -4 + -4 + -4 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 + 1 + 1 + 1 + 1 + 1 + 1 - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - 16 - 32 + + none - 900 - 900 - 900 - 900 - 900 - 900 - 900 - 900 + -16 + -16 + -16 + -16 + -16 + -16 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 + 1 + 1 + 1 + 1 + 1 + 1 - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - + none - 1800 - 1800 - 1800 - 1800 - 1800 - 1800 - 1800 - 1800 + -8 + -8 + -8 + -8 + -8 + -8 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 + 1 + 1 + 1 + 1 + 1 + 1 - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - + none - - 1200 - 1200 - 1200 - 1200 - 1200 - 1200 - 1200 - 1200 + + -8 + -8 + -8 + -8 + -8 + -8 - 3 - 3 - 3 - 3 - 3 - 3 - 3 - 3 + 1 + 1 + 1 + 1 + 1 + 1 - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - + - - - none + + - -16 - -16 - -16 - -16 - -16 - -16 - -16 - -16 + -100 + -100 + -100 + -100 + -100 + -100 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 + 1 + 1 + 1 + 1 + 1 + 1 - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - + none - 4800 - 4800 - 4800 - 4800 - 4800 - 4800 - 4800 - 4800 + -16 + -16 + -16 + -16 + -16 + -16 + -16 + -16 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 - + none - 4800 - 4800 - 4800 - 4800 - 4800 - 4800 - 4800 - 4800 + 4800 + 4800 + 4800 + 4800 + 4800 + 4800 + 4800 + 4800 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 + 4 + 4 + 4 + 4 + 4 + 4 + 4 + 4 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 @@ -626,34 +466,34 @@ none - 16384 - 16384 - 16384 - 16384 - 16384 - 16384 - 16384 - 16384 + 16384 + 16384 + 16384 + 16384 + 16384 + 16384 + 16384 + 16384 - 8 - 8 - 8 - 8 - 8 - 8 - 8 - 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 @@ -663,73 +503,34 @@ none - 8192 - 8192 - 8192 - 8192 - 8192 - 8192 - 8192 - 8192 - - - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - 16 - 32 - none - - 7680 - 3040 - 3040 - 3840 - 7680 - 7680 - 7680 - 7680 + 8192 + 8192 + 8192 + 8192 + 8192 + 8192 + 8192 + 8192 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 - 0 - 0 - 0 - 3040 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 @@ -739,295 +540,205 @@ none - -32 - -32 - -32 - -32 - -32 - -32 - -32 - -32 + -32 + -32 + -32 + -32 + -32 + -32 + -32 + -32 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 - - - - 32 - 32 + + + none - -1 - -1 - -1 - -1 - -1 - -1 - -1 - -1 + 128 + 128 + 128 + 128 + 128 + 128 + 128 + 128 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 - - - - - + none - 12 - 12 - 12 - 12 - 12 - 12 - 12 - 12 + -4 + -4 + -4 + -4 + -4 + -4 + -4 + -4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - + + + none - 64 - 64 - 64 - 64 - 64 - 64 - 64 - 64 + -2 + -2 + -2 + -2 + -2 + -2 + -2 + -2 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 - - + + none - -2 - -2 - -2 - -2 - -2 - -2 - -2 - -2 + 224 + 224 + 224 + 224 + 224 + 224 + 224 + 224 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 - + none - 96 - 96 - 96 - 96 - 96 - 96 - 96 - 96 + 192 + 192 + 192 + 192 + 192 + 192 + 192 + 192 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 224 - 224 - 224 - 224 - 224 - 224 - 224 - 224 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 192 - 192 - 192 - 192 - 192 - 192 - 192 - 192 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 @@ -1037,34 +748,34 @@ none - 192 - 192 - 192 - 192 - 192 - 192 - 192 - 192 + 192 + 192 + 192 + 192 + 192 + 192 + 192 + 192 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 @@ -1074,62 +785,61 @@ none - 240 - 240 - 240 - 240 - 240 - 240 - 240 - 240 + 240 + 240 + 240 + 240 + 240 + 240 + 240 + 240 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 - - + + - 36 none - 360 - 360 - 360 - 360 - 360 - 360 - 360 - 360 + 36 + 36 + 36 + 36 + 36 + 36 + 36 + 36 - 3 - 3 - 3 - 3 - 3 - 3 - 3 - 3 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 0 @@ -1144,29 +854,53 @@ - - + + none - 144 - 144 - 144 - 144 - 144 - 144 - 144 - 144 + -8 + -8 + -8 + -8 + -8 + -8 + + + 1 + 1 + 1 + 1 + 1 + 1 + + + + + + + + + none + + -4 + -4 + -4 + -4 + -4 + -4 + -4 + -4 - 3 - 3 - 3 - 3 - 3 - 3 - 3 - 3 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 0 @@ -1179,27 +913,31 @@ 0 - + + + + + none - 288 - 288 - 288 - 288 - 288 - 288 - 288 - 288 + -208 + -208 + -208 + -208 + -208 + -208 + -208 + -208 - 3 - 3 - 3 - 3 - 3 - 3 - 3 - 3 + 8 + 8 + 8 + 8 + 8 + 8 + 8 + 8 0 @@ -1213,65 +951,53 @@ - - + + + + none - 192 - 192 - 192 - 192 - 192 - 192 - 192 - 192 + -8 + -8 + -8 + -8 + -8 + -8 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 + 1 + 1 + 1 + 1 + 1 + 1 - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - + - - - + + + none - 288 - 288 - 288 - 288 - 288 - 288 - 288 - 288 + -8 + -8 + -8 + -8 + -8 + -8 + -8 + -8 - 3 - 3 - 3 - 3 - 3 - 3 - 3 - 3 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 0 @@ -1286,67 +1012,66 @@ - - - + + - 16 - 32 none - 192 - 192 - 192 - 192 - 192 - 192 - 192 - 192 + 480 + 480 + 480 + 480 + 480 + 480 + 480 + 480 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 - - 16 - 32 + + + + + none - 256 - 256 - 256 - 256 - 256 - 256 - 256 - 256 + -16 + -16 + -16 + -16 + -16 + -16 + -16 + -16 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 0 @@ -1359,29 +1084,31 @@ 0 - - 16 - 32 + + + + + none - 128 - 128 - 128 - 128 - 128 - 128 - 128 - 128 + 960 + 960 + 960 + 960 + 960 + 960 + 960 + 960 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 0 @@ -1396,415 +1123,441 @@ - + + - - none + - -4 - -4 - -4 - -4 - -4 - -4 - -4 - -4 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - + + + - none - - 192 - 192 - 192 - 192 - 192 - 192 - 192 - 192 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - + none + + 36 + 36 + 36 + 36 + 36 + 36 + 36 + 36 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + - - + + + + - none - - -208 - -208 - -208 - -208 - -208 - -208 - -208 - -208 - - - 8 - 8 - 8 - 8 - 8 - 8 - 8 - 8 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - + none + + -1 + -1 + -1 + -1 + -1 + -1 + -1 + -1 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + - - + + + - none - - -64 - -64 - -64 - -64 - -64 - -64 - -64 - -64 - - - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - + none + + -4 + -4 + -4 + -4 + -4 + -4 + -4 + -4 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + - - - - none - - 384 - 384 - 384 - 384 - 384 - 384 - 384 - 384 - - - 3 - 3 - 3 - 3 - 3 - 3 - 3 - 3 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - + + + + + none + + -3 + -3 + -3 + -3 + -3 + -3 + -3 + -3 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + - - - + - none - - -8 - -8 - -8 - -8 - -8 - -8 - -8 - -8 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - + none + + -16 + -16 + -16 + -16 + -16 + -16 + -16 + -16 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + - - - none - - 768 - 768 - 768 - 768 - 768 - 768 - 768 - 768 - - - 3 - 3 - 3 - 3 - 3 - 3 - 3 - 3 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - + + + + + + none + + -32 + -32 + -32 + -32 + -32 + -32 + -32 + -32 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + - - + + + - none - - 480 - 480 - 480 - 480 - 480 - 480 - 480 - 480 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - + none + + -32 + -32 + -32 + -32 + -32 + -32 + -32 + -32 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + - - + + + + + - none - - -16 - -16 - -16 - -16 - -16 - -16 - -16 - -16 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - + none + + 1920 + 1920 + 1920 + 1920 + 1920 + 1920 + 1920 + 1920 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + - - + + + - none - - 960 - 960 - 960 - 960 - 960 - 960 - 960 - 960 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - + none + + -118 + -118 + -118 + -118 + -118 + -118 + -118 + -118 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + - - - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - + + + + none + + -135 + -135 + -135 + -135 + -135 + -135 + -135 + -135 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - - - 1 - - - 1 - - + + none + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + diff --git a/cime_config/namelist_definition_cam.xml b/cime_config/namelist_definition_cam.xml index fa4440e4..847b762a 100644 --- a/cime_config/namelist_definition_cam.xml +++ b/cime_config/namelist_definition_cam.xml @@ -139,7 +139,7 @@ ${DIN_LOC_ROOT}/atm/waccm/ic/fw2000_ne30np4_L70_c181221.nc - /glade/campaign/cgd/amp/aherring/mpas-uniform/mpasa120/grids/mpasa120_L26_notopo_coords_c251105.nc + ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa120_L26_notopo_coords_c251105.nc ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa480_L32_notopo_coords_c240507.nc ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa120_L32_notopo_coords_c240507.nc ${DIN_LOC_ROOT}/atm/cam/inic/mpas/mpasa60_L32_notopo_coords_c240507.nc @@ -467,6 +467,7 @@ 30 32 32 + 26 diff --git a/src/physics/utils/phys_comp.F90 b/src/physics/utils/phys_comp.F90 index 142902de..471ee9e0 100644 --- a/src/physics/utils/phys_comp.F90 +++ b/src/physics/utils/phys_comp.F90 @@ -202,11 +202,13 @@ subroutine phys_timestep_init() use cam_abortutils, only: endrun use cam_ccpp_cap, only: cam_ccpp_physics_timestep_initial use time_manager, only: is_first_step + use runtime_obj, only: cam_runtime_opts ! Local variables - type(file_desc_t), pointer :: ncdata - integer :: data_frame - logical :: use_init_variables + type(file_desc_t), pointer :: ncdata + integer :: data_frame + logical :: use_init_variables + logical :: is_null_dycore ! Physics needs to read in all data not read in by the dycore ncdata => initial_file_get_id() @@ -218,12 +220,26 @@ subroutine phys_timestep_init() ! Initialize host model variables that must be done each time step: call physics_types_tstep_init() + ! Determine if we are running with the null dycore, which needs to + ! read from the IC file every time step: + is_null_dycore = (cam_runtime_opts%get_dycore() == 'null') + ! Determine if we should read initialized variables from file use_init_variables = (.not. is_first_step()) .and. & (.not. is_first_restart_step()) - call physics_read_data(ncdata, suite_names, data_frame, & - read_initialized_variables=use_init_variables) + ! Read physics data from IC file. For the null dycore + ! this should be done every timestep, but for all other + ! dycores this should only be done at the first timestep: + if (.not. use_init_variables) then + !First time step, so always call: + call physics_read_data(ncdata, suite_names, data_frame, & + read_initialized_variables=use_init_variables) + else if(is_null_dycore) then + !Using null dycore, so call every time step: + call physics_read_data(ncdata, suite_names, data_frame, & + read_initialized_variables=use_init_variables) + end if ! Initialize the physics time step call cam_ccpp_physics_timestep_initial(phys_suite_name) diff --git a/src/physics/utils/physics_data.F90 b/src/physics/utils/physics_data.F90 index d0adeedf..d1dd49f1 100644 --- a/src/physics/utils/physics_data.F90 +++ b/src/physics/utils/physics_data.F90 @@ -157,6 +157,8 @@ subroutine read_field_2d(file, std_name, var_names, timestep, buffer, mark_as_re use cam_pio_utils, only: cam_pio_find_var use cam_abortutils, only: endrun use cam_logfile, only: iulog + use cam_logfile, only: debug_output + use cam_logfile, only: DEBUGOUT_INFO use cam_field_read, only: cam_read_field use phys_vars_init_check, only: mark_as_read_from_file @@ -221,7 +223,7 @@ subroutine read_field_2d(file, std_name, var_names, timestep, buffer, mark_as_re call endrun(strerr) end if else if (.not. error_on_not_found_local) then - if (masterproc) then + if (masterproc .and. (debug_output >= DEBUGOUT_INFO)) then write(iulog, *) trim(std_name), ' not found, also looked for: ', trim(arr2str(var_names)) call shr_sys_flush(iulog) end if @@ -242,6 +244,8 @@ subroutine read_field_3d(file, std_name, var_names, vcoord_name, & use cam_pio_utils, only: cam_pio_find_var use cam_abortutils, only: endrun use cam_logfile, only: iulog + use cam_logfile, only: debug_output + use cam_logfile, only: DEBUGOUT_INFO use cam_field_read, only: cam_read_field use vert_coord, only: pver, pverp use phys_vars_init_check, only: mark_as_read_from_file @@ -318,7 +322,7 @@ subroutine read_field_3d(file, std_name, var_names, vcoord_name, & call endrun(strerr) end if else if (.not. error_on_not_found_local) then - if (masterproc) then + if (masterproc .and. (debug_output >= DEBUGOUT_INFO)) then write(iulog, *) trim(std_name), ' not found, also looked for: ', trim(arr2str(var_names)) call shr_sys_flush(iulog) end if From e2ea6f38ddb35bb5b99b4c8b591adb5754f40578 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 1 Dec 2025 09:30:51 -0700 Subject: [PATCH 137/150] Replace stub CAM4 regression test with aquaplanet CAM4 test. --- cime_config/testdefs/testlist_cam.xml | 8 +++++--- .../shell_commands | 0 .../user_nl_cam | 0 3 files changed, 5 insertions(+), 3 deletions(-) rename cime_config/testdefs/testmods_dirs/cam/{outfrq_se_cslam_analy_ic_cam4 => outfrq_analy_ic_cam4}/shell_commands (100%) rename cime_config/testdefs/testmods_dirs/cam/{outfrq_se_cslam_analy_ic_cam4 => outfrq_analy_ic_cam4}/user_nl_cam (100%) diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index 1562fba9..2645c6f6 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -192,14 +192,16 @@ - + + + - - + + diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_se_cslam_analy_ic_cam4/shell_commands b/cime_config/testdefs/testmods_dirs/cam/outfrq_analy_ic_cam4/shell_commands similarity index 100% rename from cime_config/testdefs/testmods_dirs/cam/outfrq_se_cslam_analy_ic_cam4/shell_commands rename to cime_config/testdefs/testmods_dirs/cam/outfrq_analy_ic_cam4/shell_commands diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_se_cslam_analy_ic_cam4/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq_analy_ic_cam4/user_nl_cam similarity index 100% rename from cime_config/testdefs/testmods_dirs/cam/outfrq_se_cslam_analy_ic_cam4/user_nl_cam rename to cime_config/testdefs/testmods_dirs/cam/outfrq_analy_ic_cam4/user_nl_cam From 1b0cf31fe26435fee1154c1b5d8064ae81ff6535 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 1 Dec 2025 09:34:01 -0700 Subject: [PATCH 138/150] Remove leftover debugging code. --- cime_config/testdefs/testlist_cam.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index 2645c6f6..81542fff 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -196,8 +196,6 @@ - - From cc27c6629804002c93f40d6a91d7d1be34a8cb86 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 2 Dec 2025 13:59:48 -0700 Subject: [PATCH 139/150] Fix bugs found after ESCOMP/CAM-SIMA gravity waves merge. --- .gitmodules | 2 +- cime_config/cam_config.py | 1 + src/data/registry.xml | 8 -------- src/physics/ncar_ccpp | 2 +- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.gitmodules b/.gitmodules index 70ca5d89..2e5d9246 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/nusbaume/atmospheric_physics - fxtag = 7d5dd0125802e38b2e5b119a36e206da95b50e02 + fxtag = e5061eed66a529389b3db9481eae1f27b06dda16 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/cime_config/cam_config.py b/cime_config/cam_config.py index 34415039..01737912 100644 --- a/cime_config/cam_config.py +++ b/cime_config/cam_config.py @@ -253,6 +253,7 @@ def __init__(self, case, case_log): #Add the default host model namelists: self._add_xml_nml_file(cime_conf_path, "namelist_definition_cam.xml") self._add_xml_nml_file(data_nml_path, "namelist_definition_physconst.xml") + self._add_xml_nml_file(data_nml_path, "namelist_definition_ref_pres.xml") self._add_xml_nml_file(cpl_nuopc_nml_path, "namelist_definition_atm_stream_ndep.xml") diff --git a/src/data/registry.xml b/src/data/registry.xml index c3f68c9d..f772694b 100644 --- a/src/data/registry.xml +++ b/src/data/registry.xml @@ -1668,14 +1668,6 @@ 0.0_kind_phys pbuf_TTEND_DP - - horizontal_dimension vertical_layer_dimension - 0.0_kind_phys - pbuf_TTEND_DP - Date: Wed, 10 Dec 2025 08:25:21 -0700 Subject: [PATCH 140/150] Update all namelist 'pg' versions to 2.0, and fix failing python unit tests. --- cime_config/cam_config.py | 8 ++++---- src/data/namelist_definition_physconst.xml | 2 +- src/data/namelist_definition_ref_pres.xml | 2 +- .../mpas/namelist_definition_mpas_dycore.xml | 2 +- .../se/namelist_definition_se_dycore.xml | 2 +- .../tests/namelist_definition_analy_ic.xml | 2 +- .../sample_files/physics_types_complete.F90 | 2 +- test/unit/python/test_cam_config.py | 16 +++++++++------- 8 files changed, 19 insertions(+), 17 deletions(-) diff --git a/cime_config/cam_config.py b/cime_config/cam_config.py index 01737912..496f09b1 100644 --- a/cime_config/cam_config.py +++ b/cime_config/cam_config.py @@ -586,25 +586,25 @@ def parse_config_opts(cls, config_opts, test_mode=False): >>> config_opts = ConfigCAM.parse_config_opts("--physics-suites kessler") >>> vargs = vars(config_opts) >>> [(x, vargs[x]) for x in sorted(vargs)] - [('analytic_ic', False), ('dyn', ''), ('dyn_kind', 'REAL64'), ('phys_kind', 'REAL64'), ('physics_suites', 'kessler')] + [('analytic_ic', False), ('aquaplanet', False), ('dyn', ''), ('dyn_kind', 'REAL64'), ('phys_kind', 'REAL64'), ('physics_suites', 'kessler')] 4. Check that parse_config_opts works as expected when given a physics suite and a second argument: >>> config_opts = ConfigCAM.parse_config_opts("--physics-suites kessler --dyn se") >>> vargs = vars(config_opts) >>> [(x, vargs[x]) for x in sorted(vargs)] - [('analytic_ic', False), ('dyn', 'se'), ('dyn_kind', 'REAL64'), ('phys_kind', 'REAL64'), ('physics_suites', 'kessler')] + [('analytic_ic', False), ('aquaplanet', False), ('dyn', 'se'), ('dyn_kind', 'REAL64'), ('phys_kind', 'REAL64'), ('physics_suites', 'kessler')] 5. Check that parse_config_opts works as expected when given both a string and logical argument: >>> config_opts = ConfigCAM.parse_config_opts("--physics-suites kessler --dyn se --analytic-ic") >>> vargs = vars(config_opts) >>> [(x, vargs[x]) for x in sorted(vargs)] - [('analytic_ic', True), ('dyn', 'se'), ('dyn_kind', 'REAL64'), ('phys_kind', 'REAL64'), ('physics_suites', 'kessler')] + [('analytic_ic', True), ('aquaplanet', False), ('dyn', 'se'), ('dyn_kind', 'REAL64'), ('phys_kind', 'REAL64'), ('physics_suites', 'kessler')] 6. Check that parse_config_opts works as expected when given multiple physics suites: >>> config_opts = ConfigCAM.parse_config_opts("--physics-suites kessler;musica") >>> vargs = vars(config_opts) >>> [(x, vargs[x]) for x in sorted(vargs)] - [('analytic_ic', False), ('dyn', ''), ('dyn_kind', 'REAL64'), ('phys_kind', 'REAL64'), ('physics_suites', 'kessler;musica')] + [('analytic_ic', False), ('aquaplanet', False), ('dyn', ''), ('dyn_kind', 'REAL64'), ('phys_kind', 'REAL64'), ('physics_suites', 'kessler;musica')] 7. Check that parse_config_opts fails correctly when given an un-recognized argument: >>> ConfigCAM.parse_config_opts("--phys kessler musica", test_mode=True) diff --git a/src/data/namelist_definition_physconst.xml b/src/data/namelist_definition_physconst.xml index 7c3e99d5..aa0a340a 100644 --- a/src/data/namelist_definition_physconst.xml +++ b/src/data/namelist_definition_physconst.xml @@ -1,6 +1,6 @@ - + diff --git a/src/data/namelist_definition_ref_pres.xml b/src/data/namelist_definition_ref_pres.xml index 15ce9e43..c42d2388 100644 --- a/src/data/namelist_definition_ref_pres.xml +++ b/src/data/namelist_definition_ref_pres.xml @@ -1,6 +1,6 @@ - + diff --git a/src/dynamics/mpas/namelist_definition_mpas_dycore.xml b/src/dynamics/mpas/namelist_definition_mpas_dycore.xml index c59570bf..5e51ee13 100644 --- a/src/dynamics/mpas/namelist_definition_mpas_dycore.xml +++ b/src/dynamics/mpas/namelist_definition_mpas_dycore.xml @@ -1,5 +1,5 @@ - + diff --git a/src/dynamics/tests/namelist_definition_analy_ic.xml b/src/dynamics/tests/namelist_definition_analy_ic.xml index f71490bc..eb9b39e9 100644 --- a/src/dynamics/tests/namelist_definition_analy_ic.xml +++ b/src/dynamics/tests/namelist_definition_analy_ic.xml @@ -1,6 +1,6 @@ - + diff --git a/test/unit/python/sample_files/physics_types_complete.F90 b/test/unit/python/sample_files/physics_types_complete.F90 index bcd25ccc..f8bf0c25 100644 --- a/test/unit/python/sample_files/physics_types_complete.F90 +++ b/test/unit/python/sample_files/physics_types_complete.F90 @@ -84,7 +84,7 @@ subroutine allocate_physics_types_complete_fields(set_init_val_in, reallocate_in use physics_grid, only: horizontal_dimension=>columns_on_task use vert_coord, only: vertical_layer_dimension=>pver - use cam_constituents, only: number_of_ccpp_constituents=>num_advected + use cam_constituents, only: number_of_ccpp_constituents=>num_constituents !! Dummy arguments logical, optional, intent(in) :: set_init_val_in diff --git a/test/unit/python/test_cam_config.py b/test/unit/python/test_cam_config.py index 74e38bee..687a197c 100644 --- a/test/unit/python/test_cam_config.py +++ b/test/unit/python/test_cam_config.py @@ -373,13 +373,9 @@ def test_print_all(self): 'DEBUG:print_config:# Switch to turn on analytic initial conditions for the dynamics state: \n#'+\ ' 0 => no \n# 1 => yes.', 'DEBUG:print_config:analytic_ic = 0', - 'DEBUG:print_config:# The ocean model being used.\n#'+\ - ' Valid values include prognostic ocean models (POP or MOM),\n#'+\ - ' data ocean models (DOCN or DOM), a stub ocean (SOCN), \n#'+\ - ' and an aqua planet ocean (aquaplanet).\n#'+\ - ' This does not impact how the case is built, only how\n#'+\ - ' attributes are matched when searching for namelist defaults.', - 'DEBUG:print_config:ocn = socn', + 'DEBUG:print_config:# Switch to use aquaplanet configuration: \n#'+\ + ' 0 => no \n# 1 => yes.', + 'DEBUG:print_config:aquaplanet = 0', "DEBUG:print_config:# A semicolon-separated list of physics suite definition file (SDF) names.\n#"+\ " To specify the Kessler and Held-Suarez suites as \n#"+\ " run time options, use '--physics-suites kessler;held_suarez_1994'.", @@ -768,6 +764,10 @@ def test_xml_nml_file(self): data_path = os.path.join(CAM_CONF_DIR, os.pardir, "src", "data") + #Create path to "src/cpl/nuopc" directory: + cpl_path = os.path.join(CAM_CONF_DIR, os.pardir, + "src", "cpl", "nuopc") + #These files will always be present: xml_fil_list['namelist_definition_cam.xml'] = os.path.join(CAM_CONF_DIR, 'namelist_definition_cam.xml') @@ -775,6 +775,8 @@ def test_xml_nml_file(self): 'namelist_definition_physconst.xml') xml_fil_list['namelist_definition_ref_pres.xml'] = os.path.join(data_path, 'namelist_definition_ref_pres.xml') + xml_fil_list['namelist_definition_atm_stream_ndep.xml'] = os.path.join(cpl_path, + 'namelist_definition_atm_stream_ndep.xml') #This is the file being added: xml_fil_list['test_file.xml'] = '/fake/path/test_file.xml' From 6ad4ee96fc81d868d3793a7df26293c11e4a3eee Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 8 Jan 2026 15:50:50 -0700 Subject: [PATCH 141/150] Update external submodules to match cam6_4_143. --- .gitmodules | 24 +++++++++--------------- ccs_config | 2 +- cime | 2 +- components/cdeps | 2 +- components/cice | 2 +- components/cmeps | 2 +- components/mizuRoute | 2 +- share | 2 +- src/physics/ncar_ccpp | 2 +- tools/CUPiD | 2 +- 10 files changed, 18 insertions(+), 24 deletions(-) diff --git a/.gitmodules b/.gitmodules index 2e5d9246..03b58ded 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/nusbaume/atmospheric_physics - fxtag = e5061eed66a529389b3db9481eae1f27b06dda16 + fxtag = 40610bd811e1cd34b2a4c2d7d5784a037d3769b6 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] @@ -32,33 +32,27 @@ [submodule "ccs_config"] path = ccs_config url = https://github.com/ESMCI/ccs_config_cesm.git - fxtag = ccs_config_cesm1.0.61 + fxtag = ccs_config_cesm1.0.65 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git [submodule "cdeps"] path = components/cdeps url = https://github.com/ESCOMP/CDEPS.git - fxtag = cdeps1.0.80 + fxtag = cdeps1.0.84 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git [submodule "cice"] path = components/cice url = https://github.com/ESCOMP/CESM_CICE - fxtag = cesm3_cice6_6_0_13 + fxtag = cesm3_cice6_6_1_6 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CESM_CICE [submodule "cime"] path = cime url = https://github.com/ESMCI/cime - fxtag = cime6.1.126 + fxtag = cime6.1.145 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESMCI/cime -[submodule "cism"] - path = components/cism - url = https://github.com/ESCOMP/CISM-wrapper - fxtag = cismwrap_2_2_010 - fxrequired = ToplevelRequired - fxDONOTUSEurl = https://github.com/ESCOMP/CISM-wrapper [submodule "clm"] path = components/clm url = https://github.com/ESCOMP/CTSM @@ -68,7 +62,7 @@ [submodule "cmeps"] path = components/cmeps url = https://github.com/ESCOMP/CMEPS.git - fxtag = cmeps1.1.16 + fxtag = cmeps1.1.24 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git [submodule "fms"] @@ -80,7 +74,7 @@ [submodule "mizuRoute"] path = components/mizuRoute url = https://github.com/ESCOMP/mizuRoute - fxtag = v3.0.0 + fxtag = v3.0.1 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute [submodule "mosart"] @@ -104,12 +98,12 @@ [submodule "share"] path = share url = https://github.com/ESCOMP/CESM_share - fxtag = share1.1.9 + fxtag = share1.1.16 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CESM_share [submodule "tools/CUPiD"] path = tools/CUPiD url = https://github.com/NCAR/CUPiD.git - fxtag = v0.3.1 + fxtag = v0.4.0 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/NCAR/CUPiD.git diff --git a/ccs_config b/ccs_config index b20a207b..7fda4a54 160000 --- a/ccs_config +++ b/ccs_config @@ -1 +1 @@ -Subproject commit b20a207bc918b956b8dad44c14c4471aff9331ee +Subproject commit 7fda4a54ec6261d2affb2351ca2b8f288308f0e8 diff --git a/cime b/cime index c0681033..0fc24bad 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit c0681033bdfb71d10821b478fe562e825f28268b +Subproject commit 0fc24bad917f692bce6438d56c573b4e9c7a6ef8 diff --git a/components/cdeps b/components/cdeps index 692150fc..7e51a85a 160000 --- a/components/cdeps +++ b/components/cdeps @@ -1 +1 @@ -Subproject commit 692150fc020e31235784634dd3eab4e78629bb82 +Subproject commit 7e51a85aa2f4a645ca6795ff63cd3eaa54e85453 diff --git a/components/cice b/components/cice index 48737cc1..b6618b88 160000 --- a/components/cice +++ b/components/cice @@ -1 +1 @@ -Subproject commit 48737cc126b3a9ea9839746a4fc1e4a17aeddb93 +Subproject commit b6618b88e9802884d8ff3a7092b9850319453413 diff --git a/components/cmeps b/components/cmeps index a0735ecd..cbc1f0fd 160000 --- a/components/cmeps +++ b/components/cmeps @@ -1 +1 @@ -Subproject commit a0735ecdb5a3e1150797a8f05ded6e4b6a271b41 +Subproject commit cbc1f0fd442066d171a973ad45d7f867f8cae6c6 diff --git a/components/mizuRoute b/components/mizuRoute index 07dfd26a..f4c3482f 160000 --- a/components/mizuRoute +++ b/components/mizuRoute @@ -1 +1 @@ -Subproject commit 07dfd26a17acb914b9036f59a8c6237990da09f2 +Subproject commit f4c3482f2b29b17ff66c00b615b3305c616fc85a diff --git a/share b/share index 14338bef..7e6d7a78 160000 --- a/share +++ b/share @@ -1 +1 @@ -Subproject commit 14338bef3fa604d49160e376257264db1d3313e5 +Subproject commit 7e6d7a7854ccb8e80683d3dcc6e41220eef01896 diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index e5061eed..40610bd8 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit e5061eed66a529389b3db9481eae1f27b06dda16 +Subproject commit 40610bd811e1cd34b2a4c2d7d5784a037d3769b6 diff --git a/tools/CUPiD b/tools/CUPiD index 3d953177..880f3b30 160000 --- a/tools/CUPiD +++ b/tools/CUPiD @@ -1 +1 @@ -Subproject commit 3d9531779b6f3e4c26059b10ba67c0f4477684e5 +Subproject commit 880f3b30997d19f1b766c483af957277866bbe74 From ec795b7ae42a59b8520aca6d64f85cc3a630a24e Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 8 Jan 2026 16:08:40 -0700 Subject: [PATCH 142/150] Remove old git-fleximod copy. --- .../git-fleximod/.github/workflows/pre-commit | 13 - .../.github/workflows/pytest.yaml | 77 -- .lib/git-fleximod/.pre-commit-config.yaml | 18 - .lib/git-fleximod/CODE_OF_CONDUCT.md | 107 --- .lib/git-fleximod/License | 20 - .lib/git-fleximod/README.md | 108 --- .lib/git-fleximod/doc/Makefile | 20 - .lib/git-fleximod/doc/conf.py | 26 - .lib/git-fleximod/doc/index.rst | 24 - .lib/git-fleximod/doc/make.bat | 35 - .lib/git-fleximod/escomp_install | 25 - .lib/git-fleximod/git_fleximod/__init__.py | 0 .lib/git-fleximod/git_fleximod/cli.py | 145 ---- .../git-fleximod/git_fleximod/git_fleximod.py | 370 ---------- .../git-fleximod/git_fleximod/gitinterface.py | 122 --- .lib/git-fleximod/git_fleximod/gitmodules.py | 97 --- .../git-fleximod/git_fleximod/lstripreader.py | 43 -- .lib/git-fleximod/git_fleximod/metoflexi.py | 236 ------ .lib/git-fleximod/git_fleximod/submodule.py | 462 ------------ .lib/git-fleximod/git_fleximod/utils.py | 365 --------- .lib/git-fleximod/poetry.lock | 693 ------------------ .lib/git-fleximod/pyproject.toml | 42 -- .lib/git-fleximod/tbump.toml | 43 -- .lib/git-fleximod/tests/__init__.py | 3 - .lib/git-fleximod/tests/conftest.py | 150 ---- .lib/git-fleximod/tests/test_a_import.py | 8 - .lib/git-fleximod/tests/test_b_update.py | 26 - .lib/git-fleximod/tests/test_c_required.py | 43 -- .lib/git-fleximod/tests/test_d_complex.py | 66 -- .../tests/test_e_complex_update.py | 69 -- 30 files changed, 3456 deletions(-) delete mode 100644 .lib/git-fleximod/.github/workflows/pre-commit delete mode 100644 .lib/git-fleximod/.github/workflows/pytest.yaml delete mode 100644 .lib/git-fleximod/.pre-commit-config.yaml delete mode 100644 .lib/git-fleximod/CODE_OF_CONDUCT.md delete mode 100644 .lib/git-fleximod/License delete mode 100644 .lib/git-fleximod/README.md delete mode 100644 .lib/git-fleximod/doc/Makefile delete mode 100644 .lib/git-fleximod/doc/conf.py delete mode 100644 .lib/git-fleximod/doc/index.rst delete mode 100644 .lib/git-fleximod/doc/make.bat delete mode 100644 .lib/git-fleximod/escomp_install delete mode 100644 .lib/git-fleximod/git_fleximod/__init__.py delete mode 100644 .lib/git-fleximod/git_fleximod/cli.py delete mode 100755 .lib/git-fleximod/git_fleximod/git_fleximod.py delete mode 100644 .lib/git-fleximod/git_fleximod/gitinterface.py delete mode 100644 .lib/git-fleximod/git_fleximod/gitmodules.py delete mode 100644 .lib/git-fleximod/git_fleximod/lstripreader.py delete mode 100755 .lib/git-fleximod/git_fleximod/metoflexi.py delete mode 100644 .lib/git-fleximod/git_fleximod/submodule.py delete mode 100644 .lib/git-fleximod/git_fleximod/utils.py delete mode 100644 .lib/git-fleximod/poetry.lock delete mode 100644 .lib/git-fleximod/pyproject.toml delete mode 100644 .lib/git-fleximod/tbump.toml delete mode 100644 .lib/git-fleximod/tests/__init__.py delete mode 100644 .lib/git-fleximod/tests/conftest.py delete mode 100644 .lib/git-fleximod/tests/test_a_import.py delete mode 100644 .lib/git-fleximod/tests/test_b_update.py delete mode 100644 .lib/git-fleximod/tests/test_c_required.py delete mode 100644 .lib/git-fleximod/tests/test_d_complex.py delete mode 100644 .lib/git-fleximod/tests/test_e_complex_update.py diff --git a/.lib/git-fleximod/.github/workflows/pre-commit b/.lib/git-fleximod/.github/workflows/pre-commit deleted file mode 100644 index 1a6ad008..00000000 --- a/.lib/git-fleximod/.github/workflows/pre-commit +++ /dev/null @@ -1,13 +0,0 @@ -name: pre-commit -on: - pull_request: - push: - branches: [main] - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: pre-commit/action@v3.0.0 diff --git a/.lib/git-fleximod/.github/workflows/pytest.yaml b/.lib/git-fleximod/.github/workflows/pytest.yaml deleted file mode 100644 index 0868dd9a..00000000 --- a/.lib/git-fleximod/.github/workflows/pytest.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# Run this job on pushes to `main`, and for pull requests. If you don't specify -# `branches: [main], then this actions runs _twice_ on pull requests, which is -# annoying. - -on: - push: - branches: [main] - pull_request: - branches: [main] - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - # If you wanted to use multiple Python versions, you'd have specify a matrix in the job and - # reference the matrixe python version here. - - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - # Cache the installation of Poetry itself, e.g. the next step. This prevents the workflow - # from installing Poetry every time, which can be slow. Note the use of the Poetry version - # number in the cache key, and the "-0" suffix: this allows you to invalidate the cache - # manually if/when you want to upgrade Poetry, or if something goes wrong. This could be - # mildly cleaner by using an environment variable, but I don't really care. - - name: cache poetry install - uses: actions/cache@v4 - with: - path: ~/.local - key: poetry-1.7.1 - - # Install Poetry. You could do this manually, or there are several actions that do this. - # `snok/install-poetry` seems to be minimal yet complete, and really just calls out to - # Poetry's default install script, which feels correct. I pin the Poetry version here - # because Poetry does occasionally change APIs between versions and I don't want my - # actions to break if it does. - # - # The key configuration value here is `virtualenvs-in-project: true`: this creates the - # venv as a `.venv` in your testing directory, which allows the next step to easily - # cache it. - - uses: snok/install-poetry@v1 - with: - version: 1.7.1 - virtualenvs-create: true - virtualenvs-in-project: true - - # Cache your dependencies (i.e. all the stuff in your `pyproject.toml`). Note the cache - # key: if you're using multiple Python versions, or multiple OSes, you'd need to include - # them in the cache key. I'm not, so it can be simple and just depend on the poetry.lock. - - name: cache deps - id: cache-deps - uses: actions/cache@v4 - with: - path: .venv - key: pydeps-${{ hashFiles('**/poetry.lock') }} - - # Install dependencies. `--no-root` means "install all dependencies but not the project - # itself", which is what you want to avoid caching _your_ code. The `if` statement - # ensures this only runs on a cache miss. - - run: poetry install --no-interaction --no-root - if: steps.cache-deps.outputs.cache-hit != 'true' - - # Now install _your_ project. This isn't necessary for many types of projects -- particularly - # things like Django apps don't need this. But it's a good idea since it fully-exercises the - # pyproject.toml and makes that if you add things like console-scripts at some point that - # they'll be installed and working. - - run: poetry install --no-interaction - - # And finally run tests. I'm using pytest and all my pytest config is in my `pyproject.toml` - # so this line is super-simple. But it could be as complex as you need. - - run: | - git config --global user.name "${GITHUB_ACTOR}" - git config --global user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" - poetry run pytest - diff --git a/.lib/git-fleximod/.pre-commit-config.yaml b/.lib/git-fleximod/.pre-commit-config.yaml deleted file mode 100644 index 2f6089da..00000000 --- a/.lib/git-fleximod/.pre-commit-config.yaml +++ /dev/null @@ -1,18 +0,0 @@ -exclude: ^utils/.*$ - -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 - hooks: - - id: end-of-file-fixer - - id: trailing-whitespace - - repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black - - repo: https://github.com/PyCQA/pylint - rev: v2.11.1 - hooks: - - id: pylint - args: - - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error,no-member diff --git a/.lib/git-fleximod/CODE_OF_CONDUCT.md b/.lib/git-fleximod/CODE_OF_CONDUCT.md deleted file mode 100644 index 84f2925b..00000000 --- a/.lib/git-fleximod/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,107 +0,0 @@ -# Contributor Code of Conduct -_The Contributor Code of Conduct is for participants in our software projects and community._ - -## Our Pledge -We, as contributors, creators, stewards, and maintainers (participants), of **git-fleximod** pledge to make participation in -our software, system or hardware project and community a safe, productive, welcoming and inclusive experience for everyone. -All participants are required to abide by this Code of Conduct. -This includes respectful treatment of everyone regardless of age, body size, disability, ethnicity, gender identity or expression, -level of experience, nationality, political affiliation, veteran status, pregnancy, genetic information, physical appearance, race, -religion, or sexual orientation, as well as any other characteristic protected under applicable US federal or state law. - -## Our Standards -Examples of behaviors that contribute to a positive environment include: - -* All participants are treated with respect and consideration, valuing a diversity of views and opinions -* Be considerate, respectful, and collaborative -* Communicate openly with respect for others, critiquing ideas rather than individuals and gracefully accepting criticism -* Acknowledging the contributions of others -* Avoid personal attacks directed toward other participants -* Be mindful of your surroundings and of your fellow participants -* Alert UCAR staff and suppliers/vendors if you notice a dangerous situation or someone in distress -* Respect the rules and policies of the project and venue - -Examples of unacceptable behavior include, but are not limited to: - -* Harassment, intimidation, or discrimination in any form -* Physical, verbal, or written abuse by anyone to anyone, including repeated use of pronouns other than those requested -* Unwelcome sexual attention or advances -* Personal attacks directed at other guests, members, participants, etc. -* Publishing others' private information, such as a physical or electronic address, without explicit permission -* Alarming, intimidating, threatening, or hostile comments or conduct -* Inappropriate use of nudity and/or sexual images -* Threatening or stalking anyone, including a participant -* Other conduct which could reasonably be considered inappropriate in a professional setting - -## Scope -This Code of Conduct applies to all spaces managed by the Project whether they be physical, online or face-to-face. -This includes project code, code repository, associated web pages, documentation, mailing lists, project websites and wiki pages, -issue tracker, meetings, telecons, events, project social media accounts, and any other forums created by the project team which the -community uses for communication. -In addition, violations of this Code of Conduct outside these spaces may affect a person's ability to participate within them. -Representation of a project may be further defined and clarified by project maintainers. - -## Community Responsibilities -Everyone in the community is empowered to respond to people who are showing unacceptable behavior. -They can talk to them privately or publicly. -Anyone requested to stop unacceptable behavior is expected to comply immediately. -If the behavior continues concerns may be brought to the project administrators or to any other party listed in the -[Reporting](#reporting) section below. - -## Project Administrator Responsibilities -Project administrators are responsible for clarifying the standards of acceptable behavior and are encouraged to model appropriate -behavior and provide support when people in the community point out inappropriate behavior. -Project administrator(s) are normally the ones that would be tasked to carry out the actions in the [Consequences](#consequences) -section below. - -Project administrators are also expected to keep this Code of Conduct updated with the main one housed at UCAR, as listed below in -the [Attribution](#attribution) section. - -## Reporting -Instances of unacceptable behavior can be brought to the attention of the project administrator(s) who may take any action as -outlined in the [Consequences](#consequences) section below. -However, making a report to a project administrator is not considered an 'official report' to UCAR. - -Instances of unacceptable behavior may also be reported directly to UCAR pursuant to [UCAR's Harassment Reporting and Complaint -Procedure](https://www2.fin.ucar.edu/procedures/hr/harassment-reporting-and-complaint-procedure), or anonymously through [UCAR's -EthicsPoint Hotline](https://www2.fin.ucar.edu/ethics/anonymous-reporting). - -Complaints received by UCAR will be handled pursuant to the procedures outlined in UCAR's Harassment Reporting and Complaint -Procedure. -Complaints to UCAR will be held as confidential as practicable under the circumstances, and retaliation against a person who -initiates a complaint or an inquiry about inappropriate behavior will not be tolerated. - -Any Contributor can use these reporting methods even if they are not directly affiliated with UCAR. -The Frequently Asked Questions (FAQ) page for reporting is [here](https://www2.fin.ucar.edu/procedures/hr/reporting-faqs). - -## Consequences -Upon receipt of a complaint, the project administrator(s) may take any action deemed necessary and appropriate under the -circumstances. -Such action can include things such as: removing, editing, or rejecting comments, commits, code, wiki edits, email, issues, and -other contributions that are not aligned to this Code of Conduct, or banning temporarily or permanently any contributor for other -behaviors that are deemed inappropriate, threatening, offensive, or harmful. -Project administrators also have the right to report violations to UCAR HR and/or UCAR's Office of Diversity, Equity and Inclusion -(ODEI), as well as a participant's home institution and/or law enforcement. -In the event an incident is reported to UCAR, UCAR will follow its Harassment Reporting and Complaint Procedure. - -## Process for Changes -All UCAR managed projects are required to adopt this Contributor Code of Conduct. -Adoption is assumed even if not expressly stated in the repository. -Projects should fill in sections where prompted with project-specific information, including, project name and adoption date. - -Projects that adopt this Code of Conduct need to stay up to date with UCAR's Contributor Code of Conduct, linked with a DOI in the -[Attribution](#attribution) section below. -Projects can make limited substantive changes to the Code of Conduct, however, the changes must be limited in scope and may not -contradict the UCAR Contributor Code of Conduct. - -## Attribution -This Code of Conduct was originally adapted from the [Contributor Covenant](http://contributor-covenant.org/version/1/4), version -1.4. -We then aligned it with the UCAR Participant Code of Conduct, which also borrows from the American Geophysical Union (AGU) Code of -Conduct. -The UCAR Participant Code of Conduct applies to both UCAR employees as well as participants in activities run by UCAR. -The original version of this for all software projects that have strong management from UCAR or UCAR staff is available on the UCAR -website at https://doi.org/10.5065/6w2c-a132. -The date that it was adopted by this project was **Feb/13/2018**. -When responding to complaints, UCAR HR and ODEI will do so based on the latest published version. -Therefore, any project-specific changes should follow the [Process for Changes](#process-for-changes) section above. diff --git a/.lib/git-fleximod/License b/.lib/git-fleximod/License deleted file mode 100644 index 88bc2251..00000000 --- a/.lib/git-fleximod/License +++ /dev/null @@ -1,20 +0,0 @@ -Copyright 2024 NSF National Center for Atmospheric Sciences (NCAR) - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -“Software”), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/.lib/git-fleximod/README.md b/.lib/git-fleximod/README.md deleted file mode 100644 index 0978a260..00000000 --- a/.lib/git-fleximod/README.md +++ /dev/null @@ -1,108 +0,0 @@ -# git-fleximod - -Flexible, Enhanced Submodule Management for Git - -## Overview - -Git-fleximod is a Python-based tool that extends Git's submodule and sparse checkout capabilities, offering additional features for managing submodules in a more flexible and efficient way. - -## Installation - - If you choose to locate git-fleximod in your path you can access it via command: git fleximod - -## Usage - - Basic Usage: - git fleximod [options] - Available Commands: - status: Display the status of submodules. - update: Update submodules to the tag indicated in .gitmodules variable fxtag. - test: Make sure that fxtags and submodule hashes are consistant, - make sure that official urls (as defined by fxDONOTUSEurl) are set - make sure that fxtags are defined for all submodules - Additional Options: - See git fleximod --help for more details. - -## Supported .gitmodules Variables - - fxtag: Specify a specific tag or hash to checkout for a submodule. Branches are not acceptable. - fxrequired: Mark a submodule's checkout behavior, with allowed values: - - ToplevelRequired: Top-level and required (checked out only when this is the Toplevel module). - - ToplevelOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module). - - AlwaysRequired: Always required (always checked out). - - AlwaysOptional: Always optional (checked out with --optional flag). - fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths. - fxDONOTUSEurl: This is the url used in the test subcommand to assure that protected branches do not point to forks - **NOTE** the fxDONOTUSEurl variable is only used to identify the official project repository and should not be - changed by users. Use the url variable to change to a fork if desired. - -## Sparse Checkouts - - To enable sparse checkout for a submodule, set the fxsparse variable - in the .gitmodules file to the path of a file containing the desired - sparse checkout paths. Git-fleximod will automatically configure - sparse checkout based on this file when applicable commands are run. - See [git-sparse-checkout](https://git-scm.com/docs/git-sparse-checkout#_internalsfull_pattern_set) - for details on the format of this file. - -## Tests - - The git fleximod test action is designed to be used by, for example, github workflows - to assure that protected branches are consistant with respect to submodule hashes and fleximod fxtags - -## Examples - -Here are some common usage examples: - -Update all submodules, including optional ones: -```bash - git fleximod update --optional -``` - -Updating a specific submodule to the fxtag indicated in .gitmodules: - -```bash - git fleximod update submodule-name -``` -Example .gitmodules entry: -```ini, toml - [submodule "cosp2"] - path = src/physics/cosp2/src - url = https://github.com/CFMIP/COSPv2.0 - fxsparse = ../.cosp_sparse_checkout - fxrequired = AlwaysRequired - fxtag = v2.1.4cesm -``` -Explanation: - -This entry indicates that the submodule named cosp2 at tag v2.1.4cesm -should be checked out into the directory src/physics/cosp2/src -relative to the .gitmodules directory. It should be checked out from -the URL https://github.com/CFMIP/COSPv2.0 and use sparse checkout as -described in the file ../.cosp_sparse_checkout relative to the path -directory. It should be checked out anytime this .gitmodules entry is -read. - -Additional example: -```ini, toml - [submodule "cime"] - path = cime - url = https://github.com/jedwards4b/cime - fxrequired = ToplevelRequired - fxtag = cime6.0.198_rme01 -``` - -Explanation: - -This entry indicates that the submodule cime should be checked out -into a directory named cime at tag cime6.0.198_rme01 from the URL -https://github.com/jedwards4b/cime. This should only be done if -the .gitmodules file is at the top level of the repository clone. - -## Contributing - -We welcome contributions! Please see the CONTRIBUTING.md file for guidelines. - -## License - -Git-fleximod is released under the MIT License. diff --git a/.lib/git-fleximod/doc/Makefile b/.lib/git-fleximod/doc/Makefile deleted file mode 100644 index d4bb2cbb..00000000 --- a/.lib/git-fleximod/doc/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/.lib/git-fleximod/doc/conf.py b/.lib/git-fleximod/doc/conf.py deleted file mode 100644 index 423099ee..00000000 --- a/.lib/git-fleximod/doc/conf.py +++ /dev/null @@ -1,26 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# For the full list of built-in configuration values, see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Project information ----------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information - -project = "git-fleximod" -author = "Jim Edwards " -release = "0.4.0" - -# -- General configuration --------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration - -extensions = ["sphinx_argparse_cli"] - -templates_path = ["_templates"] -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - - -# -- Options for HTML output ------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output - -html_theme = "alabaster" -html_static_path = ["_static"] diff --git a/.lib/git-fleximod/doc/index.rst b/.lib/git-fleximod/doc/index.rst deleted file mode 100644 index 0f9c1a7f..00000000 --- a/.lib/git-fleximod/doc/index.rst +++ /dev/null @@ -1,24 +0,0 @@ -.. git-fleximod documentation master file, created by - sphinx-quickstart on Sat Feb 3 12:02:22 2024. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to git-fleximod's documentation! -======================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: -.. module:: sphinxcontrib.autoprogram -.. sphinx_argparse_cli:: - :module: git_fleximod.cli - :func: get_parser - :prog: git-fleximod - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/.lib/git-fleximod/doc/make.bat b/.lib/git-fleximod/doc/make.bat deleted file mode 100644 index 32bb2452..00000000 --- a/.lib/git-fleximod/doc/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "" goto help - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/.lib/git-fleximod/escomp_install b/.lib/git-fleximod/escomp_install deleted file mode 100644 index ae782e72..00000000 --- a/.lib/git-fleximod/escomp_install +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# updates git-fleximod in an ESCOMP model -# this script should be run from the model root directory, it expects -# git-fleximod to already be installed with the script in bin -# and the classes in lib/python/site-packages -import sys -import shutil -import os - -from glob import iglob - -fleximod_root = sys.argv[1] -fleximod_path = os.path.join(fleximod_root,"src","git-fleximod") -if os.path.isfile(fleximod_path): - with open(fleximod_path,"r") as f: - fleximod = f.readlines() - with open(os.path.join(".","bin","git-fleximod"),"w") as f: - for line in fleximod: - f.write(line) - if "import argparse" in line: - f.write('\nsys.path.append(os.path.join(os.path.dirname(__file__),"..","lib","python","site-packages"))\n\n') - - for file in iglob(os.path.join(fleximod_root, "src", "fleximod", "*.py")): - shutil.copy(file, - os.path.join("lib","python","site-packages","fleximod",os.path.basename(file))) diff --git a/.lib/git-fleximod/git_fleximod/__init__.py b/.lib/git-fleximod/git_fleximod/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/.lib/git-fleximod/git_fleximod/cli.py b/.lib/git-fleximod/git_fleximod/cli.py deleted file mode 100644 index e4acc555..00000000 --- a/.lib/git-fleximod/git_fleximod/cli.py +++ /dev/null @@ -1,145 +0,0 @@ -from pathlib import Path -import argparse, os, sys -from git_fleximod import utils - -__version__ = "1.0.3" - -class CustomArgumentParser(argparse.ArgumentParser): - def print_help(self, file=None): - # First print the default help message - super().print_help(file) - - # Then append the contents of README.md - candidate_paths = [ - Path(sys.prefix) / "share" / "git_fleximod" / "README.md", - Path(__file__).resolve().parent.parent / "README.md", # fallback for dev - ] - for path in candidate_paths: - if os.path.exists(path): - with open(path) as f: - print( f.read(), file=file) - return - print( "README.md not found.", file=file) - -def find_root_dir(filename=".gitmodules"): - """ finds the highest directory in tree - which contains a file called filename """ - d = Path.cwd() - root = Path(d.root) - dirlist = [] - dl = d - while dl != root: - dirlist.append(dl) - dl = dl.parent - dirlist.append(root) - dirlist.reverse() - - for dl in dirlist: - attempt = dl / filename - if attempt.is_file(): - return str(dl) - return None - -def get_parser(): - description = """ - %(prog)s manages checking out groups of gitsubmodules with additional support for Earth System Models - """ - parser = CustomArgumentParser( - description=description, formatter_class=argparse.RawDescriptionHelpFormatter - ) - - # - # user options - # - choices = ["update", "status", "test"] - parser.add_argument( - "action", - choices=choices, - default="update", - help=f"Subcommand of git-fleximod, choices are {choices[:-1]}", - ) - - parser.add_argument( - "components", - nargs="*", - help="Specific component(s) to checkout. By default, " - "all required submodules are checked out.", - ) - - parser.add_argument( - "-C", - "--path", - default=find_root_dir(), - help="Toplevel repository directory. Defaults to top git directory relative to current.", - ) - - parser.add_argument( - "-g", - "--gitmodules", - nargs="?", - default=".gitmodules", - help="The submodule description filename. " "Default: %(default)s.", - ) - - parser.add_argument( - "-x", - "--exclude", - nargs="*", - help="Component(s) listed in the gitmodules file which should be ignored.", - ) - parser.add_argument( - "-f", - "--force", - action="store_true", - default=False, - help="Override cautions and update or checkout over locally modified repository.", - ) - - parser.add_argument( - "-o", - "--optional", - action="store_true", - default=False, - help="By default only the required submodules " - "are checked out. This flag will also checkout the " - "optional submodules relative to the toplevel directory.", - ) - - parser.add_argument( - "-v", - "--verbose", - action="count", - default=0, - help="Output additional information to " - "the screen and log file. This flag can be " - "used up to two times, increasing the " - "verbosity level each time.", - ) - - parser.add_argument( - "-V", - "--version", - action="version", - version=f"%(prog)s {__version__}", - help="Print version and exit.", - ) - - # - # developer options - # - parser.add_argument( - "--backtrace", - action="store_true", - help="DEVELOPER: show exception backtraces as extra " "debugging output", - ) - - parser.add_argument( - "-d", - "--debug", - action="store_true", - default=False, - help="DEVELOPER: output additional debugging " - "information to the screen and log file.", - ) - - return parser diff --git a/.lib/git-fleximod/git_fleximod/git_fleximod.py b/.lib/git-fleximod/git_fleximod/git_fleximod.py deleted file mode 100755 index b3c4fece..00000000 --- a/.lib/git-fleximod/git_fleximod/git_fleximod.py +++ /dev/null @@ -1,370 +0,0 @@ -#!/usr/bin/env python -import sys - -MIN_PYTHON = (3, 7) -if sys.version_info < MIN_PYTHON: - sys.exit("Python %s.%s or later is required." % MIN_PYTHON) - -import os -import shutil -import logging -import textwrap -import asyncio -from git_fleximod import utils -from git_fleximod import cli -from git_fleximod.gitinterface import GitInterface -from git_fleximod.gitmodules import GitModules -from git_fleximod.submodule import Submodule - -# logger variable is global -logger = None - - -def fxrequired_allowed_values(): - return ["ToplevelRequired", "ToplevelOptional", "AlwaysRequired", "AlwaysOptional", "TopLevelRequired", "TopLevelOptional"] - - -def commandline_arguments(args=None): - parser = cli.get_parser() - - if args: - options = parser.parse_args(args) - else: - options = parser.parse_args() - - # explicitly listing a component overrides the optional flag - if options.optional or options.components: - fxrequired = fxrequired_allowed_values() - else: - fxrequired = ["ToplevelRequired", "AlwaysRequired", "TopLevelRequired"] - - action = options.action - if not action: - action = "update" - handlers = [logging.StreamHandler()] - - if options.debug: - try: - open("fleximod.log", "w") - except PermissionError: - sys.exit("ABORT: Could not write file fleximod.log") - level = logging.DEBUG - handlers.append(logging.FileHandler("fleximod.log")) - elif options.verbose: - level = logging.INFO - else: - level = logging.WARNING - # Configure the root logger - logging.basicConfig( - level=level, format="%(name)s - %(levelname)s - %(message)s", handlers=handlers - ) - - if hasattr(options, "version"): - exit() - - return ( - options.path, - options.gitmodules, - fxrequired, - options.components, - options.exclude, - options.force, - action, - ) - - -def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master"): - """ - This function performs a sparse checkout of a git submodule. It does so by first creating the .git/info/sparse-checkout fileq - in the submodule and then checking out the desired tag. If the submodule is already checked out, it will not be checked out again. - Creating the sparse-checkout file first prevents the entire submodule from being checked out and then removed. This is important - because the submodule may have a large number of files and checking out the entire submodule and then removing it would be time - and disk space consuming. - - Parameters: - root_dir (str): The root directory for the git operation. - name (str): The name of the submodule. - url (str): The URL of the submodule. - path (str): The path to the submodule. - sparsefile (str): The sparse file for the submodule. - tag (str, optional): The tag to checkout. Defaults to "master". - - Returns: - None - """ - logger.info("Called sparse_checkout for {}".format(name)) - rgit = GitInterface(root_dir, logger) - superroot = git_toplevelroot(root_dir, logger) - - if superroot: - gitroot = superroot.strip() - else: - gitroot = root_dir.strip() - assert os.path.isdir(os.path.join(gitroot, ".git")) - # first create the module directory - if not os.path.isdir(os.path.join(root_dir, path)): - os.makedirs(os.path.join(root_dir, path)) - - # initialize a new git repo and set the sparse checkout flag - sprep_repo = os.path.join(root_dir, path) - sprepo_git = GitInterface(sprep_repo, logger) - if os.path.exists(os.path.join(sprep_repo, ".git")): - try: - logger.info("Submodule {} found".format(name)) - chk = sprepo_git.config_get_value("core", "sparseCheckout") - if chk == "true": - logger.info("Sparse submodule {} already checked out".format(name)) - return - except NoOptionError: - logger.debug("Sparse submodule {} not present".format(name)) - except Exception as e: - utils.fatal_error("Unexpected error {} occured.".format(e)) - - sprepo_git.config_set_value("core", "sparseCheckout", "true") - - # set the repository remote - - logger.info("Setting remote origin in {}/{}".format(root_dir, path)) - _, remotelist = sprepo_git.git_operation("remote", "-v") - if url not in remotelist: - sprepo_git.git_operation("remote", "add", "origin", url) - - topgit = os.path.join(gitroot, ".git") - - if gitroot != root_dir and os.path.isfile(os.path.join(root_dir, ".git")): - with open(os.path.join(root_dir, ".git")) as f: - gitpath = os.path.relpath( - os.path.join(root_dir, f.read().split()[1]), - start=os.path.join(root_dir, path), - ) - topgit = os.path.join(gitpath, "modules") - else: - topgit = os.path.relpath( - os.path.join(root_dir, ".git", "modules"), - start=os.path.join(root_dir, path), - ) - - with utils.pushd(sprep_repo): - if not os.path.isdir(topgit): - os.makedirs(topgit) - topgit += os.sep + name - - if os.path.isdir(os.path.join(root_dir, path, ".git")): - with utils.pushd(sprep_repo): - if os.path.isdir(os.path.join(topgit,".git")): - shutil.rmtree(os.path.join(topgit,".git")) - shutil.move(".git", topgit) - with open(".git", "w") as f: - f.write("gitdir: " + os.path.relpath(topgit)) - # assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo))) - gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout")) - if os.path.isfile(gitsparse): - logger.warning( - "submodule {} is already initialized {}".format(name, topgit) - ) - return - - with utils.pushd(sprep_repo): - if os.path.isfile(sparsefile): - shutil.copy(sparsefile, gitsparse) - - - # Finally checkout the repo - sprepo_git.git_operation("fetch", "origin", "--tags") - sprepo_git.git_operation("checkout", tag) - - print(f"Successfully checked out {name:>20} at {tag}") - rgit.config_set_value(f'submodule "{name}"', "active", "true") - rgit.config_set_value(f'submodule "{name}"', "url", url) - -def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger): - path = gitmodules.get(name, "path") - url = gitmodules.get(name, "url") - assert path and url, f"Malformed .gitmodules file {path} {url}" - tag = gitmodules.get(name, "fxtag") - if not tag: - tag = gitmodules.get(name, "hash") - fxurl = gitmodules.get(name, "fxDONOTUSEurl") - fxsparse = gitmodules.get(name, "fxsparse") - fxrequired = gitmodules.get(name, "fxrequired") - return Submodule(root_dir, name, path, url, fxtag=tag, fxurl=fxurl, fxsparse=fxsparse, fxrequired=fxrequired, logger=logger) - -def submodules_status(gitmodules, root_dir, toplevel=False, depth=0): - testfails = 0 - localmods = 0 - needsupdate = 0 - wrapper = textwrap.TextWrapper(initial_indent=' '*(depth*10), width=120,subsequent_indent=' '*(depth*20)) - for name in gitmodules.sections(): - submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) - - result,n,l,t = submod.status() - if toplevel or not submod.toplevel(): - print(wrapper.fill(result)) - testfails += t - localmods += l - needsupdate += n - subdir = os.path.join(root_dir, submod.path) - if os.path.exists(os.path.join(subdir, ".gitmodules")): - gsubmod = GitModules(logger, confpath=subdir) - t,l,n = submodules_status(gsubmod, subdir, depth=depth+1) - if toplevel or not submod.toplevel(): - testfails += t - localmods += l - needsupdate += n - - return testfails, localmods, needsupdate - -def git_toplevelroot(root_dir, logger): - rgit = GitInterface(root_dir, logger) - _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") - return superroot - -async def submodules_update(gitmodules, root_dir, requiredlist, force): - async def update_submodule(name, requiredlist, force): - submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) - - _, needsupdate, localmods, testfails = submod.status() - if not submod.fxrequired: - submod.fxrequired = "AlwaysRequired" - fxrequired = submod.fxrequired - allowedvalues = fxrequired_allowed_values() - assert fxrequired in allowedvalues - - superroot = git_toplevelroot(root_dir, logger) - - if ( - fxrequired - and ((superroot and "Toplevel" in fxrequired) - or fxrequired not in requiredlist) - ): - if "Optional" in fxrequired and "Optional" not in requiredlist: - if fxrequired.startswith("Always"): - print(f"Skipping optional component {name:>20}") - return # continue to next submodule - optional = "AlwaysOptional" in requiredlist - - if fxrequired in requiredlist: - await submod.update() - repodir = os.path.join(root_dir, submod.path) - if os.path.exists(os.path.join(repodir, ".gitmodules")): - # recursively handle this checkout - print(f"Recursively checking out submodules of {name}") - gitsubmodules = GitModules(submod.logger, confpath=repodir) - newrequiredlist = ["AlwaysRequired"] - if optional: - newrequiredlist.append("AlwaysOptional") - await submodules_update(gitsubmodules, repodir, newrequiredlist, force=force) - - tasks = [update_submodule(name, requiredlist, force) for name in gitmodules.sections()] - await asyncio.gather(*tasks) - -def local_mods_output(): - text = """\ - The submodules labeled with 'M' above are not in a clean state. - The following are options for how to proceed: - (1) Go into each submodule which is not in a clean state and issue a 'git status' - Either revert or commit your changes so that the submodule is in a clean state. - (2) use the --force option to git-fleximod - (3) you can name the particular submodules to update using the git-fleximod command line - (4) As a last resort you can remove the submodule (via 'rm -fr [directory]') - then rerun git-fleximod update. -""" - print(text) - -def submodules_test(gitmodules, root_dir): - """ - This function tests the git submodules based on the provided parameters. - - It first checks that fxtags are present and in sync with submodule hashes. - Then it ensures that urls are consistent with fxurls (not forks and not ssh) - and that sparse checkout files exist. - - Parameters: - gitmodules (ConfigParser): The gitmodules configuration. - root_dir (str): The root directory for the git operation. - - Returns: - int: The number of test failures. - """ - # First check that fxtags are present and in sync with submodule hashes - testfails, localmods, needsupdate = submodules_status(gitmodules, root_dir) - print("") - # Then make sure that urls are consistant with fxurls (not forks and not ssh) - # and that sparse checkout files exist - for name in gitmodules.sections(): - url = gitmodules.get(name, "url") - fxurl = gitmodules.get(name, "fxDONOTUSEurl") - fxsparse = gitmodules.get(name, "fxsparse") - path = gitmodules.get(name, "path") - fxurl = fxurl[:-4] if fxurl.endswith(".git") else fxurl - url = url[:-4] if url.endswith(".git") else url - if not fxurl or url.lower() != fxurl.lower(): - print(f"{name:>20} url {url} not in sync with required {fxurl}") - testfails += 1 - if fxsparse and not os.path.isfile(os.path.join(root_dir, path, fxsparse)): - print(f"{name:>20} sparse checkout file {fxsparse} not found") - testfails += 1 - return testfails + localmods + needsupdate - - -def main(): - ( - root_dir, - file_name, - fxrequired, - includelist, - excludelist, - force, - action, - ) = commandline_arguments() - # Get a logger for the package - global logger - logger = logging.getLogger(__name__) - - logger.info("action is {} root_dir={} file_name={}".format(action, root_dir, file_name)) - - if not root_dir or not os.path.isfile(os.path.join(root_dir, file_name)): - if root_dir: - file_path = utils.find_upwards(root_dir, file_name) - - if root_dir is None or file_path is None: - root_dir = "." - utils.fatal_error( - "No {} found in {} or any of it's parents".format(file_name, root_dir) - ) - - root_dir = os.path.dirname(file_path) - logger.info( - "root_dir is {} includelist={} excludelist={}".format( - root_dir, includelist, excludelist - ) - ) - gitmodules = GitModules( - logger, - confpath=root_dir, - conffile=file_name, - includelist=includelist, - excludelist=excludelist, - ) - if not gitmodules.sections(): - sys.exit(f"No submodule components found, root_dir={root_dir}") - retval = 0 - if action == "update": - asyncio.run(submodules_update(gitmodules, root_dir, fxrequired, force)) - elif action == "status": - tfails, lmods, updates = submodules_status(gitmodules, root_dir, toplevel=True) - if tfails + lmods + updates > 0: - print( - f" testfails = {tfails}, local mods = {lmods}, needs updates {updates}\n" - ) - if lmods > 0: - local_mods_output() - elif action == "test": - retval = submodules_test(gitmodules, root_dir) - else: - utils.fatal_error(f"unrecognized action request {action}") - return retval - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.lib/git-fleximod/git_fleximod/gitinterface.py b/.lib/git-fleximod/git_fleximod/gitinterface.py deleted file mode 100644 index 022426d2..00000000 --- a/.lib/git-fleximod/git_fleximod/gitinterface.py +++ /dev/null @@ -1,122 +0,0 @@ -import os -import sys -from . import utils -from pathlib import Path -import asyncio - -class GitInterface: - def __init__(self, repo_path, logger): - logger.debug("Initialize GitInterface for {}".format(repo_path)) - if isinstance(repo_path, str): - self.repo_path = Path(repo_path).resolve() - elif isinstance(repo_path, Path): - self.repo_path = repo_path.resolve() - else: - raise TypeError("repo_path must be a str or Path object") - self.logger = logger - try: - import git - - self._use_module = True - try: - self.repo = git.Repo(str(self.repo_path)) # Initialize GitPython repo - except git.exc.InvalidGitRepositoryError: - self.git = git - self._init_git_repo() - msg = "Using GitPython interface to git" - except ImportError: - self._use_module = False - if not (self.repo_path / ".git").exists(): - self._init_git_repo() - msg = "Using shell interface to git" - self.logger.info(msg) - - def _git_command(self, operation, *args): - self.logger.info(operation) - if self._use_module and operation != "submodule": - try: - return getattr(self.repo.git, operation)(*args) - except Exception as e: - sys.exit(e) - else: - return ["git", "-C", str(self.repo_path), operation] + list(args) - - def _init_git_repo(self): - if self._use_module: - self.repo = self.git.Repo.init(str(self.repo_path)) - else: - command = ("git", "-C", str(self.repo_path), "init") - utils.execute_subprocess(command) - - def _git_operation_command(self, operation, args): - newargs = [] - for a in args: - # Do not use ssh interface - if isinstance(a, str): - a = a.replace("git@github.com:", "https://github.com/") - newargs.append(a) - - return self._git_command(operation, *newargs) - - # pylint: disable=unused-argument - def git_operation(self, operation, *args, **kwargs): - newargs = [] - for a in args: - # Do not use ssh interface - if isinstance(a, str): - a = a.replace("git@github.com:", "https://github.com/") - newargs.append(a) - - command = self._git_command(operation, *newargs) - if isinstance(command, list): - try: - status, output = utils.execute_subprocess(command, status_to_caller=True, output_to_caller=True) - return status, output.rstrip() - except Exception as e: - sys.exit(e) - else: - return 0, command - - # pylint: disable=unused-argument - async def git_operation_async(self, operation, *args, **kwargs): - command = self._git_operation_command(operation, args) - if isinstance(command, list): - try: - process = await asyncio.create_subprocess_exec( - *command, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE - ) - stdout, stderr = await process.communicate() - status = process.returncode - output = stdout.decode().strip() if stdout else stderr.decode().strip() - return status, output - except Exception as e: - sys.exit(e) - else: - return 0, command - - def config_get_value(self, section, name): - if self._use_module: - config = self.repo.config_reader() - try: - val = config.get_value(section, name) - except: - val = None - return val - else: - cmd = ("git", "-C", str(self.repo_path), "config", "--get", f"{section}.{name}") - output = utils.execute_subprocess(cmd, output_to_caller=True) - return output.strip() - - def config_set_value(self, section, name, value): - if self._use_module: - with self.repo.config_writer() as writer: - if "." in section: - section = section.replace("."," \"")+'"' - writer.set_value(section, name, value) - writer.release() # Ensure changes are saved - else: - cmd = ("git", "-C", str(self.repo_path), "config", f"{section}.{name}", value) - self.logger.info(cmd) - utils.execute_subprocess(cmd, output_to_caller=True) diff --git a/.lib/git-fleximod/git_fleximod/gitmodules.py b/.lib/git-fleximod/git_fleximod/gitmodules.py deleted file mode 100644 index cf8b350d..00000000 --- a/.lib/git-fleximod/git_fleximod/gitmodules.py +++ /dev/null @@ -1,97 +0,0 @@ -import shutil, os -from pathlib import Path -from configparser import RawConfigParser, ConfigParser -from .lstripreader import LstripReader - - -class GitModules(RawConfigParser): - def __init__( - self, - logger, - confpath=Path.cwd(), - conffile=".gitmodules", - includelist=None, - excludelist=None, - ): - """ - confpath: Path to the directory containing the .gitmodules file (defaults to the current working directory). - conffile: Name of the configuration file (defaults to .gitmodules). - includelist: Optional list of submodules to include. - excludelist: Optional list of submodules to exclude. - """ - self.logger = logger - self.logger.debug( - "Creating a GitModules object {} {} {} {}".format( - confpath, conffile, includelist, excludelist - ) - ) - super().__init__() - self.conf_file = (Path(confpath) / Path(conffile)) - if self.conf_file.exists(): - self.read_file(LstripReader(str(self.conf_file)), source=conffile) - self.includelist = includelist - self.excludelist = excludelist - self.isdirty = False - - def reload(self): - self.clear() - if self.conf_file.exists(): - self.read_file(LstripReader(str(self.conf_file)), source=self.conf_file) - - - def set(self, name, option, value): - """ - Sets a configuration value for a specific submodule: - Ensures the appropriate section exists for the submodule. - Calls the parent class's set method to store the value. - """ - self.isdirty = True - self.logger.debug("set called {} {} {}".format(name, option, value)) - section = f'submodule "{name}"' - if not self.has_section(section): - self.add_section(section) - super().set(section, option, str(value)) - - # pylint: disable=redefined-builtin, arguments-differ - def get(self, name, option, raw=False, vars=None, fallback=None): - """ - Retrieves a configuration value for a specific submodule: - Uses the parent class's get method to access the value. - Handles potential errors if the section or option doesn't exist. - """ - self.logger.debug("git get called {} {}".format(name, option)) - section = f'submodule "{name}"' - try: - return ConfigParser.get( - self, section, option, raw=raw, vars=vars, fallback=fallback - ) - except ConfigParser.NoOptionError: - return None - - def save(self): - if self.isdirty: - self.logger.info("Writing {}".format(self.conf_file)) - with open(self.conf_file, "w") as fd: - self.write(fd) - self.isdirty = False - - def __del__(self): - self.save() - - def sections(self): - """Strip the submodule part out of section and just use the name""" - self.logger.debug("calling GitModules sections iterator") - names = [] - for section in ConfigParser.sections(self): - name = section[11:-1] - if self.includelist and name not in self.includelist: - continue - if self.excludelist and name in self.excludelist: - continue - names.append(name) - return names - - def items(self, name, raw=False, vars=None): - self.logger.debug("calling GitModules items for {}".format(name)) - section = f'submodule "{name}"' - return ConfigParser.items(section, raw=raw, vars=vars) diff --git a/.lib/git-fleximod/git_fleximod/lstripreader.py b/.lib/git-fleximod/git_fleximod/lstripreader.py deleted file mode 100644 index 01d5580e..00000000 --- a/.lib/git-fleximod/git_fleximod/lstripreader.py +++ /dev/null @@ -1,43 +0,0 @@ -class LstripReader(object): - "LstripReader formats .gitmodules files to be acceptable for configparser" - - def __init__(self, filename): - with open(filename, "r") as infile: - lines = infile.readlines() - self._lines = list() - self._num_lines = len(lines) - self._index = 0 - for line in lines: - self._lines.append(line.lstrip()) - - def readlines(self): - """Return all the lines from this object's file""" - return self._lines - - def readline(self, size=-1): - """Format and return the next line or raise StopIteration""" - try: - line = self.next() - except StopIteration: - line = "" - - if (size > 0) and (len(line) < size): - return line[0:size] - - return line - - def __iter__(self): - """Begin an iteration""" - self._index = 0 - return self - - def next(self): - """Return the next line or raise StopIteration""" - if self._index >= self._num_lines: - raise StopIteration - - self._index = self._index + 1 - return self._lines[self._index - 1] - - def __next__(self): - return self.next() diff --git a/.lib/git-fleximod/git_fleximod/metoflexi.py b/.lib/git-fleximod/git_fleximod/metoflexi.py deleted file mode 100755 index cc347db2..00000000 --- a/.lib/git-fleximod/git_fleximod/metoflexi.py +++ /dev/null @@ -1,236 +0,0 @@ -#!/usr/bin/env python -from configparser import ConfigParser -import sys -import shutil -from pathlib import Path -import argparse -import logging -from git_fleximod.gitinterface import GitInterface -from git_fleximod.gitmodules import GitModules -from git_fleximod import utils - -logger = None - -def find_root_dir(filename=".git"): - d = Path.cwd() - root = Path(d.root) - while d != root: - attempt = d / filename - if attempt.is_dir(): - return d - d = d.parent - return None - - -def get_parser(): - description = """ - %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models - """ - parser = argparse.ArgumentParser( - description=description, formatter_class=argparse.RawDescriptionHelpFormatter - ) - - parser.add_argument('-e', '--externals', nargs='?', - default='Externals.cfg', - help='The externals description filename. ' - 'Default: %(default)s.') - - parser.add_argument( - "-C", - "--path", - default=find_root_dir(), - help="Toplevel repository directory. Defaults to top git directory relative to current.", - ) - - parser.add_argument( - "-g", - "--gitmodules", - nargs="?", - default=".gitmodules", - help="The submodule description filename. " "Default: %(default)s.", - ) - parser.add_argument( - "-v", - "--verbose", - action="count", - default=0, - help="Output additional information to " - "the screen and log file. This flag can be " - "used up to two times, increasing the " - "verbosity level each time.", - ) - parser.add_argument( - "-d", - "--debug", - action="store_true", - default=False, - help="DEVELOPER: output additional debugging " - "information to the screen and log file.", - ) - - return parser - -def commandline_arguments(args=None): - parser = get_parser() - - options = parser.parse_args(args) - handlers = [logging.StreamHandler()] - - if options.debug: - try: - open("fleximod.log", "w") - except PermissionError: - sys.exit("ABORT: Could not write file fleximod.log") - level = logging.DEBUG - handlers.append(logging.FileHandler("fleximod.log")) - elif options.verbose: - level = logging.INFO - else: - level = logging.WARNING - # Configure the root logger - logging.basicConfig( - level=level, format="%(name)s - %(levelname)s - %(message)s", handlers=handlers - ) - - return( - options.path, - options.gitmodules, - options.externals - ) - -class ExternalRepoTranslator: - """ - Translates external repositories configured in an INI-style externals file. - """ - - def __init__(self, rootpath, gitmodules, externals): - self.rootpath = rootpath - if gitmodules: - self.gitmodules = GitModules(logger, confpath=rootpath) - self.externals = (rootpath / Path(externals)).resolve() - print(f"Translating {self.externals}") - self.git = GitInterface(rootpath, logger) - -# def __del__(self): -# if (self.rootpath / "save.gitignore"): - - - def translate_single_repo(self, section, tag, url, path, efile, hash_, sparse, protocol): - """ - Translates a single repository based on configuration details. - - Args: - rootpath (str): Root path of the main repository. - gitmodules (str): Path to the .gitmodules file. - tag (str): The tag to use for the external repository. - url (str): The URL of the external repository. - path (str): The relative path within the main repository for the external repository. - efile (str): The external file or file containing submodules. - hash_ (str): The commit hash to checkout (if applicable). - sparse (str): Boolean indicating whether to use sparse checkout (if applicable). - protocol (str): The protocol to use (e.g., 'git', 'http'). - """ - assert protocol != "svn", "SVN protocol is not currently supported" - print(f"Translating repository {section}") - if efile: - file_path = Path(path) / Path(efile) - newroot = (self.rootpath / file_path).parent.resolve() - if not newroot.exists(): - newroot.mkdir(parents=True) - logger.info("Newroot is {}".format(newroot)) - newt = ExternalRepoTranslator(newroot, ".gitmodules", efile) - newt.translate_repo() - if protocol == "externals_only": - if tag: - self.gitmodules.set(section, "fxtag", tag) - if hash_: - self.gitmodules.set(section, "fxtag", hash_) - - self.gitmodules.set(section, "fxDONOTUSEurl", url) - if sparse: - self.gitmodules.set(section, "fxsparse", sparse) - self.gitmodules.set(section, "fxrequired", "ToplevelRequired") - else: - newpath = (self.rootpath / Path(path)) - if newpath.exists(): - shutil.rmtree(newpath) - logger.info("Creating directory {}".format(newpath)) - newpath.mkdir(parents=True) - if tag: - logger.info("cloning {}".format(section)) - try: - self.git.git_operation("clone", "-b", tag, "--depth", "1", url, path) - except: - self.git.git_operation("clone", url, path) - with utils.pushd(newpath): - ngit = GitInterface(newpath, logger) - ngit.git_operation("checkout", tag) - if hash_: - self.git.git_operation("clone", url, path) - git = GitInterface(newpath, logger) - git.git_operation("fetch", "origin") - git.git_operation("checkout", hash_) - if sparse: - print("setting as sparse submodule {}".format(section)) - sparsefile = (newpath / Path(sparse)) - newfile = (newpath / ".git" / "info" / "sparse-checkout") - print(f"sparsefile {sparsefile} newfile {newfile}") - shutil.copy(sparsefile, newfile) - - logger.info("adding submodule {}".format(section)) - self.gitmodules.save() - self.git.git_operation("submodule", "add", "-f", "--name", section, url, path) - self.git.git_operation("submodule","absorbgitdirs") - self.gitmodules.reload() - if tag: - self.gitmodules.set(section, "fxtag", tag) - if hash_: - self.gitmodules.set(section, "fxtag", hash_) - - self.gitmodules.set(section, "fxDONOTUSEurl", url) - if sparse: - self.gitmodules.set(section, "fxsparse", sparse) - self.gitmodules.set(section, "fxrequired", "ToplevelRequired") - - - def translate_repo(self): - """ - Translates external repositories defined within an external file. - - Args: - rootpath (str): Root path of the main repository. - gitmodules (str): Path to the .gitmodules file. - external_file (str): The path to the external file containing repository definitions. - """ - econfig = ConfigParser() - econfig.read((self.rootpath / Path(self.externals))) - - for section in econfig.sections(): - if section == "externals_description": - logger.info("skipping section {}".format(section)) - return - logger.info("Translating section {}".format(section)) - tag = econfig.get(section, "tag", raw=False, fallback=None) - url = econfig.get(section, "repo_url", raw=False, fallback=None) - path = econfig.get(section, "local_path", raw=False, fallback=None) - efile = econfig.get(section, "externals", raw=False, fallback=None) - hash_ = econfig.get(section, "hash", raw=False, fallback=None) - sparse = econfig.get(section, "sparse", raw=False, fallback=None) - protocol = econfig.get(section, "protocol", raw=False, fallback=None) - - self.translate_single_repo(section, tag, url, path, efile, hash_, sparse, protocol) - - - -def _main(): - rootpath, gitmodules, externals = commandline_arguments() - global logger - logger = logging.getLogger(__name__) - with utils.pushd(rootpath): - t = ExternalRepoTranslator(Path(rootpath), gitmodules, externals) - logger.info("Translating {}".format(rootpath)) - t.translate_repo() - - -if __name__ == "__main__": - sys.exit(_main()) diff --git a/.lib/git-fleximod/git_fleximod/submodule.py b/.lib/git-fleximod/git_fleximod/submodule.py deleted file mode 100644 index 5a8c3f93..00000000 --- a/.lib/git-fleximod/git_fleximod/submodule.py +++ /dev/null @@ -1,462 +0,0 @@ -import os -import textwrap -import shutil -import string -from configparser import NoOptionError -from git_fleximod import utils -from git_fleximod.gitinterface import GitInterface - -class Submodule(): - """ - Represents a Git submodule with enhanced features for flexible management. - - Attributes: - name (str): The name of the submodule. - root_dir (str): The root directory of the main project. - path (str): The relative path from the root directory to the submodule. - url (str): The URL of the submodule repository. - fxurl (str): The URL for flexible submodule management (optional). - fxtag (str): The tag for flexible submodule management (optional). - fxsparse (str): Path to the sparse checkout file relative to the submodule path, see git-sparse-checkout for details (optional). - fxrequired (str): Indicates if the submodule is optional or required (optional). - logger (logging.Logger): Logger instance for logging (optional). - """ - def __init__(self, root_dir, name, path, url, fxtag=None, fxurl=None, fxsparse=None, fxrequired=None, logger=None): - """ - Initializes a new Submodule instance with the provided attributes. - """ - self.name = name - self.root_dir = root_dir - self.path = path - self.url = url - self.fxurl = fxurl - self.fxtag = fxtag - self.fxsparse = fxsparse - if fxrequired: - self.fxrequired = fxrequired - else: - self.fxrequired = "AlwaysRequired" - self.logger = logger - - def status(self): - """ - Checks the status of the submodule and returns 4 parameters: - - result (str): The status of the submodule. - - needsupdate (bool): An indicator if the submodule needs to be updated. - - localmods (bool): An indicator if the submodule has local modifications. - - testfails (bool): An indicator if the submodule has failed a test, this is used for testing purposes. - """ - - smpath = os.path.join(self.root_dir, self.path) - testfails = False - localmods = False - needsupdate = False - ahash = None - optional = "" - if "Optional" in self.fxrequired: - optional = " (optional)" - required = None - level = None - if not os.path.exists(os.path.join(smpath, ".git")): - rootgit = GitInterface(self.root_dir, self.logger) - # submodule commands use path, not name - status, tags = rootgit.git_operation("ls-remote", "--tags", self.url) - status, result = rootgit.git_operation("submodule","status",smpath) - result = result.split() - - if result: - ahash = result[0][1:] - hhash = None - atag = None - for htag in tags.split("\n"): - if htag.endswith('^{}'): - htag = htag[:-3] - if ahash and not atag and ahash in htag: - atag = (htag.split()[1])[10:] - if self.fxtag and not hhash and htag.endswith(self.fxtag): - hhash = htag.split()[0] - if hhash and atag: - break - if self.fxtag and (ahash == hhash or atag == self.fxtag): - result = f"e {self.name:>20} not checked out, aligned at tag {self.fxtag}{optional}" - needsupdate = True - elif self.fxtag: - status, ahash = rootgit.git_operation( - "submodule", "status", "{}".format(self.path) - ) - ahash = ahash[1 : len(self.fxtag) + 1] - if self.fxtag == ahash: - result = f"e {self.name:>20} not checked out, aligned at hash {ahash}{optional}" - else: - result = f"e {self.name:>20} not checked out, out of sync at tag {atag}, expected tag is {self.fxtag}{optional}" - testfails = True - needsupdate = True - else: - result = f"e {self.name:>20} has no fxtag defined in .gitmodules{optional}" - testfails = False - else: - with utils.pushd(smpath): - git = GitInterface(smpath, self.logger) - status, remote = git.git_operation("remote") - if remote == '': - result = f"e {self.name:>20} has no associated remote" - testfails = True - needsupdate = True - return result, needsupdate, localmods, testfails - status, rurl = git.git_operation("ls-remote","--get-url") - status, lines = git.git_operation("log", "--pretty=format:\"%h %d\"") - line = lines.partition('\n')[0] - parts = line.split() - ahash = parts[0][1:] - atag = None - if len(parts) > 3: - idx = 0 - while idx < len(parts)-1: - idx = idx+1 - if parts[idx] == 'tag:': - atag = parts[idx+1] - while atag.endswith(')') or atag.endswith(',') or atag.endswith("\""): - atag = atag[:-1] - if atag == self.fxtag: - break - recurse = False - if rurl != self.url: - remote = self._add_remote(git) - git.git_operation("fetch", remote) - # Asked for a tag and found that tag - if self.fxtag and atag == self.fxtag: - result = f" {self.name:>20} at tag {self.fxtag}" - recurse = True - testfails = False - # Asked for and found a hash - elif self.fxtag and (ahash[: len(self.fxtag)] == self.fxtag or (self.fxtag.find(ahash)==0)): - result = f" {self.name:>20} at hash {ahash}" - recurse = True - testfails = False - # Asked for and found a hash - elif atag == ahash: - result = f" {self.name:>20} at hash {ahash}" - recurse = True - # Did not find requested tag or hash - elif self.fxtag: - result = f"s {self.name:>20} {atag} {ahash} is out of sync with .gitmodules {self.fxtag}" - testfails = True - needsupdate = True - else: - if atag: - result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {atag}" - else: - result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {ahash}" - testfails = False - - status, output = git.git_operation("status", "--ignore-submodules", "-uno") - if "nothing to commit" not in output: - localmods = True - result = "M" + textwrap.indent(output, " ") -# print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}") - return result, needsupdate, localmods, testfails - - - def _add_remote(self, git): - """ - Adds a new remote to the submodule if it does not already exist. - - This method checks the existing remotes of the submodule. If the submodule's URL is not already listed as a remote, - it attempts to add a new remote. The name for the new remote is generated dynamically to avoid conflicts. If no - remotes exist, it defaults to naming the new remote 'origin'. - - Args: - git (GitInterface): An instance of GitInterface to perform git operations. - - Returns: - str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL. - """ - status, remotes = git.git_operation("remote", "-v") - remotes = remotes.splitlines() - upstream = None - if remotes: - status, upstream = git.git_operation("ls-remote", "--get-url") - newremote = "newremote.00" - tmpurl = self.url.replace("git@github.com:", "https://github.com/") - line = next((s for s in remotes if self.url in s or tmpurl in s), None) - if line: - newremote = line.split()[0] - return newremote - else: - i = 0 - while newremote in remotes: - i = i + 1 - newremote = f"newremote.{i:02d}" - else: - newremote = "origin" - git.git_operation("remote", "add", newremote, self.url) - return newremote - - def toplevel(self): - """ - Returns True if the submodule is Toplevel (either Required or Optional) - """ - return True if "Top" in self.fxrequired else False - - def sparse_checkout(self): - """ - Performs a sparse checkout of the submodule. - - This method optimizes the checkout process by only checking out files specified in the submodule's sparse-checkout configuration, - rather than the entire submodule content. It achieves this by first ensuring the `.git/info/sparse-checkout` file is created and - configured in the submodule's directory. Then, it proceeds to checkout the desired tag. If the submodule has already been checked out, - this method will not perform the checkout again. - - This approach is particularly beneficial for submodules with a large number of files, as it significantly reduces the time and disk space - required for the checkout process by avoiding the unnecessary checkout and subsequent removal of unneeded files. - - Returns: - None - """ - self.logger.info("Called sparse_checkout for {}".format(self.name)) - rgit = GitInterface(self.root_dir, self.logger) - status, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") - if superroot: - gitroot = superroot.strip() - else: - gitroot = self.root_dir - # Now need to move the .git dir to the submodule location - rootdotgit = os.path.join(self.root_dir, ".git") - while os.path.isfile(rootdotgit): - with open(rootdotgit) as f: - line = f.readline().rstrip() - if line.startswith("gitdir: "): - rootdotgit = os.path.abspath(os.path.join(self.root_dir,line[8:])) - assert os.path.isdir(rootdotgit) - # first create the module directory - if not os.path.isdir(os.path.join(self.root_dir, self.path)): - os.makedirs(os.path.join(self.root_dir, self.path)) - - # initialize a new git repo and set the sparse checkout flag - sprep_repo = os.path.join(self.root_dir, self.path) - sprepo_git = GitInterface(sprep_repo, self.logger) - if os.path.exists(os.path.join(sprep_repo, ".git")): - try: - self.logger.info("Submodule {} found".format(self.name)) - chk = sprepo_git.config_get_value("core", "sparseCheckout") - if chk == "true": - self.logger.info("Sparse submodule {} already checked out".format(self.name)) - return - except (NoOptionError): - self.logger.debug("Sparse submodule {} not present".format(self.name)) - except Exception as e: - utils.fatal_error("Unexpected error {} occured.".format(e)) - - sprepo_git.config_set_value("core", "sparseCheckout", "true") - - # set the repository remote - - self.logger.info("Setting remote origin in {}/{}".format(self.root_dir, self.path)) - status, remotes = sprepo_git.git_operation("remote", "-v") - if self.url not in remotes: - sprepo_git.git_operation("remote", "add", "origin", self.url) - - topgit = os.path.join(gitroot, ".git") - - if gitroot != self.root_dir and os.path.isfile(os.path.join(self.root_dir, ".git")): - with open(os.path.join(self.root_dir, ".git")) as f: - gitpath = os.path.relpath( - os.path.join(self.root_dir, f.read().split()[1]), - start=os.path.join(self.root_dir, self.path), - ) - rootdotgit = os.path.join(gitpath, "modules", self.name) - else: - rootdotgit = os.path.relpath( - os.path.join(self.root_dir, ".git", "modules", self.name), - start=os.path.join(self.root_dir, self.path), - ) - - if os.path.isdir(os.path.join(self.root_dir, self.path, ".git")): - with utils.pushd(sprep_repo): - if os.path.isdir(os.path.join(rootdotgit,".git")): - shutil.rmtree(os.path.join(rootdotgit,".git")) - shutil.move(".git", rootdotgit) - with open(".git", "w") as f: - f.write("gitdir: " + os.path.relpath(rootdotgit)) - infodir = os.path.join(rootdotgit, "info") - if not os.path.isdir(infodir): - os.makedirs(infodir) - gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout")) - if os.path.isfile(gitsparse): - self.logger.warning( - "submodule {} is already initialized {}".format(self.name, rootdotgit) - ) - os.remove(gitsparse) - - if os.path.isfile(self.fxsparse): - shutil.copy(self.fxsparse, gitsparse) - else: - self.logger.warning( - "submodule {} could not find {}".format(self.name, self.fxsparse) - ) - - # Finally checkout the repo - sprepo_git.git_operation("fetch", "origin", "--tags") - status,_ = sprepo_git.git_operation("checkout", self.fxtag) - if status: - print(f"Error checking out {self.name:>20} at {self.fxtag}") - else: - print(f"Successfully checked out {self.name:>20} at {self.fxtag}") - status,f = sprepo_git.git_operation("status") - # Restore any files deleted from sandbox - for line in f.splitlines(): - if "deleted:" in line: - deleted_file = line.split("deleted:")[1].strip() - sprepo_git.git_operation("checkout", deleted_file) - - rgit.config_set_value('submodule.' + self.name, "active", "true") - rgit.config_set_value('submodule.' + self.name, "url", self.url) - rgit.config_set_value('submodule.' + self.name, "path", self.path) - - async def update(self): - """ - Updates the submodule to the latest or specified version. - - This method handles the update process of the submodule, including checking out the submodule into the specified path, - handling sparse checkouts if configured, and updating the submodule's URL if necessary. It supports both SSH and HTTPS URLs, - automatically converting SSH URLs to HTTPS to avoid issues for users without SSH keys. - - The update process involves the following steps: - 1. If the submodule is configured for sparse checkout, it performs a sparse checkout. - 2. If the submodule is not already checked out, it clones the submodule using the provided URL. - 3. If a specific tag or hash is provided, it checks out that tag; otherwise, it checks out the latest version. - 4. If the root `.git` is a file (indicating a submodule or a worktree), additional steps are taken to integrate the submodule properly. - - Args: - None - Note: - - SSH URLs are automatically converted to HTTPS to accommodate users without SSH keys. - - Returns: - None - """ - git = GitInterface(self.root_dir, self.logger) - repodir = os.path.join(self.root_dir, self.path) - self.logger.info("Checkout {} into {}/{}".format(self.name, self.root_dir, self.path)) - # if url is provided update to the new url - tag = None - repo_exists = False - if os.path.exists(os.path.join(repodir, ".git")): - self.logger.info("Submodule {} already checked out".format(self.name)) - repo_exists = True - # Look for a .gitmodules file in the newly checkedout repo - if self.fxsparse: - print(f"Sparse checkout {self.name} fxsparse {self.fxsparse}") - if not os.path.isfile(self.fxsparse): - self.logger.info("Submodule {} fxsparse file not found".format(self.name)) - - self.sparse_checkout() - else: - if not repo_exists and self.url: - # ssh urls cause problems for those who dont have git accounts with ssh keys defined - # but cime has one since e3sm prefers ssh to https, because the .gitmodules file was - # opened with a GitModules object we don't need to worry about restoring the file here - # it will be done by the GitModules class - if self.url.startswith("git@"): - git.git_operation("clone", self.url, self.path) - smgit = GitInterface(repodir, self.logger) - if not tag: - status, tag = smgit.git_operation("describe", "--tags", "--always") - smgit.git_operation("checkout", tag) - # Now need to move the .git dir to the submodule location - rootdotgit = os.path.join(self.root_dir, ".git") - if os.path.isfile(rootdotgit): - with open(rootdotgit) as f: - line = f.readline() - if line.startswith("gitdir: "): - rootdotgit = line[8:] - - newpath = os.path.abspath(os.path.join(self.root_dir, rootdotgit, "modules", self.name)) - if os.path.exists(newpath): - shutil.rmtree(os.path.join(repodir, ".git")) - else: - shutil.move(os.path.join(repodir, ".git"), newpath) - - with open(os.path.join(repodir, ".git"), "w") as f: - f.write("gitdir: " + os.path.relpath(newpath, start=repodir)) - - if not os.path.exists(repodir): - parent = os.path.dirname(repodir) - if not os.path.isdir(parent): - os.makedirs(parent) - git.git_operation("submodule", "add", "--name", self.name, "--", self.url, self.path) - - if not repo_exists: - git.git_operation("submodule", "init", "--", self.path) - await git.git_operation_async("submodule", "update", "--", self.path) - - if self.fxtag: - smgit = GitInterface(repodir, self.logger) - newremote = self._add_remote(smgit) - # Trying to distingush a tag from a hash - allowed = set(string.digits + 'abcdef') - status = 0 - if not set(self.fxtag) <= allowed: - # This is a tag - tag = f"refs/tags/{self.fxtag}:refs/tags/{self.fxtag}" - status,_ = smgit.git_operation("fetch", newremote, tag) - else: - # This is likely a hash, so fetch full history just in case - status,_ = smgit.git_operation("fetch", newremote) - if status == 0: - status,_ = smgit.git_operation("checkout", self.fxtag) - if status: - utils.fatal_error( - f"Failed to checkout {self.name} at tag or hash {self.fxtag} from {repodir}" - ) - - if not os.path.exists(os.path.join(repodir, ".git")): - utils.fatal_error( - f"Failed to checkout {self.name} {repo_exists} {repodir} {self.path}" - ) - - - if os.path.exists(os.path.join(self.path, ".git")): - submoddir = os.path.join(self.root_dir, self.path) - with utils.pushd(submoddir): - git = GitInterface(submoddir, self.logger) - # first make sure the url is correct - newremote = self._add_remote(git) - status, tags = git.git_operation("tag", "-l") - fxtag = self.fxtag - if fxtag and fxtag not in tags: - git.git_operation("fetch", newremote, "--tags") - status, atag = git.git_operation("describe", "--tags", "--always") - status, files = git.git_operation("diff", "--name-only", "-z") - modfiles = [] - moddirs = [] - if files: - for f in files.split('\0'): - if f: - if os.path.exists(f): - git.git_operation("checkout",f) - elif os.path.isdir(f): - moddirs.append(f) - else: - modfiles.append(f) - if fxtag and fxtag != atag: - try: - status, _ = git.git_operation("checkout", fxtag) - if not status: - print(f"{self.name:>20} updated to {fxtag}") - except Exception as error: - print(error) - - - elif not fxtag: - print(f"No fxtag found for submodule {self.name:>20}") - elif modfiles: - print(f"{self.name:>20} has modified files: {modfiles}") - elif moddirs: - print(f"{self.name:>20} has modified directories: {moddirs}") - else: - print(f"{self.name:>20} up to date.") - - - - return diff --git a/.lib/git-fleximod/git_fleximod/utils.py b/.lib/git-fleximod/git_fleximod/utils.py deleted file mode 100644 index c4f43d52..00000000 --- a/.lib/git-fleximod/git_fleximod/utils.py +++ /dev/null @@ -1,365 +0,0 @@ -#!/usr/bin/env python3 -""" -Common public utilities for manic package - -""" - -import logging -import os -import subprocess -import sys -from threading import Timer -from pathlib import Path - -LOCAL_PATH_INDICATOR = "." -# --------------------------------------------------------------------- -# -# functions to massage text for output and other useful utilities -# -# --------------------------------------------------------------------- -from contextlib import contextmanager - - -@contextmanager -def pushd(new_dir): - """context for chdir. usage: with pushd(new_dir)""" - previous_dir = os.getcwd() - os.chdir(new_dir) - try: - yield - finally: - os.chdir(previous_dir) - - -def log_process_output(output): - """Log each line of process output at debug level so it can be - filtered if necessary. By default, output is a single string, and - logging.debug(output) will only put log info heading on the first - line. This makes it hard to filter with grep. - - """ - output = output.split("\n") - for line in output: - logging.debug(line) - - -def printlog(msg, **kwargs): - """Wrapper script around print to ensure that everything printed to - the screen also gets logged. - - """ - logging.info(msg) - if kwargs: - print(msg, **kwargs) - else: - print(msg) - sys.stdout.flush() - - -def find_upwards(root_dir, filename): - """Find a file in root dir or any of it's parents""" - d = Path(root_dir) - root = Path(d.root) - while d != root: - attempt = d / filename - if attempt.exists(): - return attempt - d = d.parent - return None - - -def last_n_lines(the_string, n_lines, truncation_message=None): - """Returns the last n lines of the given string - - Args: - the_string: str - n_lines: int - truncation_message: str, optional - - Returns a string containing the last n lines of the_string - - If truncation_message is provided, the returned string begins with - the given message if and only if the string is greater than n lines - to begin with. - """ - - lines = the_string.splitlines(True) - if len(lines) <= n_lines: - return_val = the_string - else: - lines_subset = lines[-n_lines:] - str_truncated = "".join(lines_subset) - if truncation_message: - str_truncated = truncation_message + "\n" + str_truncated - return_val = str_truncated - - return return_val - - -def indent_string(the_string, indent_level): - """Indents the given string by a given number of spaces - - Args: - the_string: str - indent_level: int - - Returns a new string that is the same as the_string, except that - each line is indented by 'indent_level' spaces. - - In python3, this can be done with textwrap.indent. - """ - - lines = the_string.splitlines(True) - padding = " " * indent_level - lines_indented = [padding + line for line in lines] - return "".join(lines_indented) - - -# --------------------------------------------------------------------- -# -# error handling -# -# --------------------------------------------------------------------- - - -def fatal_error(message): - """ - Error output function - """ - logging.error(message) - raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message)) - - -# --------------------------------------------------------------------- -# -# Data conversion / manipulation -# -# --------------------------------------------------------------------- -def str_to_bool(bool_str): - """Convert a sting representation of as boolean into a true boolean. - - Conversion should be case insensitive. - """ - value = None - str_lower = bool_str.lower() - if str_lower in ("true", "t"): - value = True - elif str_lower in ("false", "f"): - value = False - if value is None: - msg = ( - 'ERROR: invalid boolean string value "{0}". ' - 'Must be "true" or "false"'.format(bool_str) - ) - fatal_error(msg) - return value - - -REMOTE_PREFIXES = ["http://", "https://", "ssh://", "git@"] - - -def is_remote_url(url): - """check if the user provided a local file path instead of a - remote. If so, it must be expanded to an absolute - path. - - """ - remote_url = False - for prefix in REMOTE_PREFIXES: - if url.startswith(prefix): - remote_url = True - return remote_url - - -def split_remote_url(url): - """check if the user provided a local file path or a - remote. If remote, try to strip off protocol info. - - """ - remote_url = is_remote_url(url) - if not remote_url: - return url - - for prefix in REMOTE_PREFIXES: - url = url.replace(prefix, "") - - if "@" in url: - url = url.split("@")[1] - - if ":" in url: - url = url.split(":")[1] - - return url - - -def expand_local_url(url, field): - """check if the user provided a local file path instead of a - remote. If so, it must be expanded to an absolute - path. - - Note: local paths of LOCAL_PATH_INDICATOR have special meaning and - represent local copy only, don't work with the remotes. - - """ - remote_url = is_remote_url(url) - if not remote_url: - if url.strip() == LOCAL_PATH_INDICATOR: - pass - else: - url = os.path.expandvars(url) - url = os.path.expanduser(url) - if not os.path.isabs(url): - msg = ( - 'WARNING: Externals description for "{0}" contains a ' - "url that is not remote and does not expand to an " - "absolute path. Version control operations may " - "fail.\n\nurl={1}".format(field, url) - ) - printlog(msg) - else: - url = os.path.normpath(url) - return url - - -# --------------------------------------------------------------------- -# -# subprocess -# -# --------------------------------------------------------------------- - -# Give the user a helpful message if we detect that a command seems to -# be hanging. -_HANGING_SEC = 300 - - -def _hanging_msg(working_directory, command): - print( - """ - -Command '{command}' -from directory {working_directory} -has taken {hanging_sec} seconds. It may be hanging. - -The command will continue to run, but you may want to abort -git-fleximod with ^C and investigate. A possible cause of hangs is git -requires authentication to access a private repository. On some -systems, git requests for authentication information will not -be displayed to the user. In this case, the program will appear to -hang. Ensure you can run git manually and access all -repositories without entering your authentication information. - -""".format( - command=command, - working_directory=working_directory, - hanging_sec=_HANGING_SEC, - ) - ) - - -def execute_subprocess(commands, status_to_caller=False, output_to_caller=False): - """Wrapper around subprocess.check_output to handle common - exceptions. - - check_output runs a command with arguments and waits - for it to complete. - - check_output raises an exception on a nonzero return code. if - status_to_caller is true, execute_subprocess returns the subprocess - return code, otherwise execute_subprocess treats non-zero return - status as an error and raises an exception. - - """ - cwd = os.getcwd() - msg = "In directory: {0}\nexecute_subprocess running command:".format(cwd) - logging.info(msg) - commands_str = " ".join(str(element) for element in commands) - logging.info(commands_str) - return_to_caller = status_to_caller or output_to_caller - status = -1 - output = "" - hanging_timer = Timer( - _HANGING_SEC, - _hanging_msg, - kwargs={"working_directory": cwd, "command": commands_str}, - ) - hanging_timer.start() - try: - output = subprocess.check_output( - commands, stderr=subprocess.STDOUT, universal_newlines=True - ) - log_process_output(output) - status = 0 - except OSError as error: - msg = failed_command_msg( - "Command execution failed. Does the executable exist?", commands - ) - logging.error(error) - fatal_error(msg) - except ValueError as error: - msg = failed_command_msg( - "DEV_ERROR: Invalid arguments trying to run subprocess", commands - ) - logging.error(error) - fatal_error(msg) - except subprocess.CalledProcessError as error: - # Only report the error if we are NOT returning to the - # caller. If we are returning to the caller, then it may be a - # simple status check. If returning, it is the callers - # responsibility determine if an error occurred and handle it - # appropriately. - msg_context = ( - "Process did not run successfully; " - "returned status {0}".format(error.returncode) - ) - msg = failed_command_msg(msg_context, commands, output=error.output) - if not return_to_caller: - logging.error(error) - logging.error(msg) - log_process_output(error.output) - fatal_error(msg) - status = error.returncode - finally: - hanging_timer.cancel() - - if status_to_caller and output_to_caller: - ret_value = (status, output) - elif status_to_caller: - ret_value = status - elif output_to_caller: - ret_value = output - else: - ret_value = None - - return ret_value - - -def failed_command_msg(msg_context, command, output=None): - """Template for consistent error messages from subprocess calls. - - If 'output' is given, it should provide the output from the failed - command - """ - - if output: - output_truncated = last_n_lines( - output, 20, truncation_message="[... Output truncated for brevity ...]" - ) - errmsg = ( - "Failed with output:\n" + indent_string(output_truncated, 4) + "\nERROR: " - ) - else: - errmsg = "" - - command_str = " ".join(command) - errmsg += """In directory - {cwd} -{context}: - {command} -""".format( - cwd=os.getcwd(), context=msg_context, command=command_str - ) - - if output: - errmsg += "See above for output from failed command.\n" - - return errmsg diff --git a/.lib/git-fleximod/poetry.lock b/.lib/git-fleximod/poetry.lock deleted file mode 100644 index ac82fb0d..00000000 --- a/.lib/git-fleximod/poetry.lock +++ /dev/null @@ -1,693 +0,0 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. - -[[package]] -name = "alabaster" -version = "0.7.13" -description = "A configurable sidebar-enabled Sphinx theme" -optional = false -python-versions = ">=3.6" -files = [ - {file = "alabaster-0.7.13-py3-none-any.whl", hash = "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3"}, - {file = "alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2"}, -] - -[[package]] -name = "babel" -version = "2.15.0" -description = "Internationalization utilities" -optional = false -python-versions = ">=3.8" -files = [ - {file = "Babel-2.15.0-py3-none-any.whl", hash = "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb"}, - {file = "babel-2.15.0.tar.gz", hash = "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"}, -] - -[package.dependencies] -pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} - -[package.extras] -dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] - -[[package]] -name = "certifi" -version = "2024.8.30" -description = "Python package for providing Mozilla's CA Bundle." -optional = false -python-versions = ">=3.6" -files = [ - {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, - {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, -] - -[[package]] -name = "charset-normalizer" -version = "3.3.2" -description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, - {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, - {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, - {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, - {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, - {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, - {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, - {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, -] - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "docutils" -version = "0.19" -description = "Docutils -- Python Documentation Utilities" -optional = false -python-versions = ">=3.7" -files = [ - {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, - {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, -] - -[[package]] -name = "exceptiongroup" -version = "1.2.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, - {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, -] - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "fsspec" -version = "2023.12.2" -description = "File-system specification" -optional = false -python-versions = ">=3.8" -files = [ - {file = "fsspec-2023.12.2-py3-none-any.whl", hash = "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"}, - {file = "fsspec-2023.12.2.tar.gz", hash = "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb"}, -] - -[package.extras] -abfs = ["adlfs"] -adl = ["adlfs"] -arrow = ["pyarrow (>=1)"] -dask = ["dask", "distributed"] -devel = ["pytest", "pytest-cov"] -dropbox = ["dropbox", "dropboxdrivefs", "requests"] -full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] -fuse = ["fusepy"] -gcs = ["gcsfs"] -git = ["pygit2"] -github = ["requests"] -gs = ["gcsfs"] -gui = ["panel"] -hdfs = ["pyarrow (>=1)"] -http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] -libarchive = ["libarchive-c"] -oci = ["ocifs"] -s3 = ["s3fs"] -sftp = ["paramiko"] -smb = ["smbprotocol"] -ssh = ["paramiko"] -tqdm = ["tqdm"] - -[[package]] -name = "gitdb" -version = "4.0.11" -description = "Git Object Database" -optional = false -python-versions = ">=3.7" -files = [ - {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, - {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, -] - -[package.dependencies] -smmap = ">=3.0.1,<6" - -[[package]] -name = "gitpython" -version = "3.1.43" -description = "GitPython is a Python library used to interact with Git repositories" -optional = false -python-versions = ">=3.7" -files = [ - {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, - {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, -] - -[package.dependencies] -gitdb = ">=4.0.1,<5" - -[package.extras] -doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] - -[[package]] -name = "idna" -version = "3.7" -description = "Internationalized Domain Names in Applications (IDNA)" -optional = false -python-versions = ">=3.5" -files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, -] - -[[package]] -name = "imagesize" -version = "1.4.1" -description = "Getting image size from png/jpeg/jpeg2000/gif file" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, - {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, -] - -[[package]] -name = "importlib-metadata" -version = "8.0.0" -description = "Read metadata from Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "importlib_metadata-8.0.0-py3-none-any.whl", hash = "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f"}, - {file = "importlib_metadata-8.0.0.tar.gz", hash = "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812"}, -] - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -perf = ["ipython"] -test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] - -[[package]] -name = "iniconfig" -version = "2.0.0" -description = "brain-dead simple config-ini parsing" -optional = false -python-versions = ">=3.7" -files = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, -] - -[[package]] -name = "jinja2" -version = "3.1.4" -description = "A very fast and expressive template engine." -optional = false -python-versions = ">=3.7" -files = [ - {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, - {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, -] - -[package.dependencies] -MarkupSafe = ">=2.0" - -[package.extras] -i18n = ["Babel (>=2.7)"] - -[[package]] -name = "markupsafe" -version = "2.1.5" -description = "Safely add untrusted strings to HTML/XML markup." -optional = false -python-versions = ">=3.7" -files = [ - {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, - {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, - {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, - {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, - {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, - {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, - {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, - {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, -] - -[[package]] -name = "packaging" -version = "24.1" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, - {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, -] - -[[package]] -name = "pluggy" -version = "1.5.0" -description = "plugin and hook calling mechanisms for python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, - {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, -] - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "pyfakefs" -version = "5.5.0" -description = "pyfakefs implements a fake file system that mocks the Python file system modules." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pyfakefs-5.5.0-py3-none-any.whl", hash = "sha256:8dbf203ab7bef1529f11f7d41b9478b898e95bf9f3b71262163aac07a518cd76"}, - {file = "pyfakefs-5.5.0.tar.gz", hash = "sha256:7448aaa07142f892d0a4eb52a5ed3206a9f02c6599e686cd97d624c18979c154"}, -] - -[[package]] -name = "pygments" -version = "2.18.0" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.8" -files = [ - {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, - {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, -] - -[package.extras] -windows-terminal = ["colorama (>=0.4.6)"] - -[[package]] -name = "pytest" -version = "8.2.2" -description = "pytest: simple powerful testing with Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "pytest-8.2.2-py3-none-any.whl", hash = "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343"}, - {file = "pytest-8.2.2.tar.gz", hash = "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=1.5,<2.0" -tomli = {version = ">=1", markers = "python_version < \"3.11\""} - -[package.extras] -dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] - -[[package]] -name = "pytz" -version = "2024.1" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -files = [ - {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, - {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, -] - -[[package]] -name = "requests" -version = "2.32.3" -description = "Python HTTP for Humans." -optional = false -python-versions = ">=3.8" -files = [ - {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, - {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, -] - -[package.dependencies] -certifi = ">=2017.4.17" -charset-normalizer = ">=2,<4" -idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" - -[package.extras] -socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] - -[[package]] -name = "smmap" -version = "5.0.1" -description = "A pure Python implementation of a sliding window memory map manager" -optional = false -python-versions = ">=3.7" -files = [ - {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, - {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, -] - -[[package]] -name = "snowballstemmer" -version = "2.2.0" -description = "This package provides 29 stemmers for 28 languages generated from Snowball algorithms." -optional = false -python-versions = "*" -files = [ - {file = "snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"}, - {file = "snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1"}, -] - -[[package]] -name = "sphinx" -version = "5.3.0" -description = "Python documentation generator" -optional = false -python-versions = ">=3.6" -files = [ - {file = "Sphinx-5.3.0.tar.gz", hash = "sha256:51026de0a9ff9fc13c05d74913ad66047e104f56a129ff73e174eb5c3ee794b5"}, - {file = "sphinx-5.3.0-py3-none-any.whl", hash = "sha256:060ca5c9f7ba57a08a1219e547b269fadf125ae25b06b9fa7f66768efb652d6d"}, -] - -[package.dependencies] -alabaster = ">=0.7,<0.8" -babel = ">=2.9" -colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} -docutils = ">=0.14,<0.20" -imagesize = ">=1.3" -importlib-metadata = {version = ">=4.8", markers = "python_version < \"3.10\""} -Jinja2 = ">=3.0" -packaging = ">=21.0" -Pygments = ">=2.12" -requests = ">=2.5.0" -snowballstemmer = ">=2.0" -sphinxcontrib-applehelp = "*" -sphinxcontrib-devhelp = "*" -sphinxcontrib-htmlhelp = ">=2.0.0" -sphinxcontrib-jsmath = "*" -sphinxcontrib-qthelp = "*" -sphinxcontrib-serializinghtml = ">=1.1.5" - -[package.extras] -docs = ["sphinxcontrib-websupport"] -lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-bugbear", "flake8-comprehensions", "flake8-simplify", "isort", "mypy (>=0.981)", "sphinx-lint", "types-requests", "types-typed-ast"] -test = ["cython", "html5lib", "pytest (>=4.6)", "typed_ast"] - -[[package]] -name = "sphinxcontrib-applehelp" -version = "1.0.4" -description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" -optional = false -python-versions = ">=3.8" -files = [ - {file = "sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e"}, - {file = "sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["pytest"] - -[[package]] -name = "sphinxcontrib-devhelp" -version = "1.0.2" -description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." -optional = false -python-versions = ">=3.5" -files = [ - {file = "sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"}, - {file = "sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["pytest"] - -[[package]] -name = "sphinxcontrib-htmlhelp" -version = "2.0.1" -description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" -optional = false -python-versions = ">=3.8" -files = [ - {file = "sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff"}, - {file = "sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["html5lib", "pytest"] - -[[package]] -name = "sphinxcontrib-jsmath" -version = "1.0.1" -description = "A sphinx extension which renders display math in HTML via JavaScript" -optional = false -python-versions = ">=3.5" -files = [ - {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, - {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, -] - -[package.extras] -test = ["flake8", "mypy", "pytest"] - -[[package]] -name = "sphinxcontrib-qthelp" -version = "1.0.3" -description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." -optional = false -python-versions = ">=3.5" -files = [ - {file = "sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72"}, - {file = "sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["pytest"] - -[[package]] -name = "sphinxcontrib-serializinghtml" -version = "1.1.5" -description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." -optional = false -python-versions = ">=3.5" -files = [ - {file = "sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952"}, - {file = "sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd"}, -] - -[package.extras] -lint = ["docutils-stubs", "flake8", "mypy"] -test = ["pytest"] - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] - -[[package]] -name = "urllib3" -version = "2.2.2" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = ">=3.8" -files = [ - {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, - {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, -] - -[package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -h2 = ["h2 (>=4,<5)"] -socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["zstandard (>=0.18.0)"] - -[[package]] -name = "wheel" -version = "0.42.0" -description = "A built-package format for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "wheel-0.42.0-py3-none-any.whl", hash = "sha256:177f9c9b0d45c47873b619f5b650346d632cdc35fb5e4d25058e09c9e581433d"}, - {file = "wheel-0.42.0.tar.gz", hash = "sha256:c45be39f7882c9d34243236f2d63cbd58039e360f85d0913425fbd7ceea617a8"}, -] - -[package.extras] -test = ["pytest (>=6.0.0)", "setuptools (>=65)"] - -[[package]] -name = "zipp" -version = "3.19.2" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.8" -files = [ - {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, - {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, -] - -[package.extras] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] - -[metadata] -lock-version = "2.0" -python-versions = "^3.8" -content-hash = "25ee2ae1d74abedde3a6637a60d4a3095ea5cf9731960875741bbc2ba84a475d" diff --git a/.lib/git-fleximod/pyproject.toml b/.lib/git-fleximod/pyproject.toml deleted file mode 100644 index 211eac1d..00000000 --- a/.lib/git-fleximod/pyproject.toml +++ /dev/null @@ -1,42 +0,0 @@ -[tool.poetry] -name = "git-fleximod" -version = "1.0.3" -description = "Extended support for git-submodule and git-sparse-checkout" -authors = ["Jim Edwards "] -maintainers = ["Jim Edwards "] -license = "MIT" -readme = "README.md" -homepage = "https://github.com/jedwards4b/git-fleximod" -keywords = ["git", "submodule", "sparse-checkout"] -packages = [ -{ include = "git_fleximod"}, -{ include = "doc"}, -{ include = "README.md"}, -] - -[tool.poetry.scripts] -git-fleximod = "git_fleximod.git_fleximod:main" -me2flexi = "git_fleximod.metoflexi:_main" -fsspec = "fsspec.fuse:main" - -[tool.poetry.dependencies] -python = "^3.8" -GitPython = "^3.1.0" -sphinx = "^5.0.0" -fsspec = "^2023.12.2" -wheel = "^0.42.0" -pytest = "^8.0.0" -pyfakefs = "^5.3.5" - -[tool.poetry.urls] -"Bug Tracker" = "https://github.com/jedwards4b/git-fleximod/issues" - -[tool.pytest.ini_options] -markers = [ - "skip_after_first: only run on first iteration" -] - -[build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" - diff --git a/.lib/git-fleximod/tbump.toml b/.lib/git-fleximod/tbump.toml deleted file mode 100644 index 20271f4d..00000000 --- a/.lib/git-fleximod/tbump.toml +++ /dev/null @@ -1,43 +0,0 @@ -# Uncomment this if your project is hosted on GitHub: -github_url = "https://github.com/jedwards4b/git-fleximod/" - -[version] -current = "1.0.3" - -# Example of a semver regexp. -# Make sure this matches current_version before -# using tbump -regex = ''' - (?P\d+) - \. - (?P\d+) - \. - (?P\d+) - ''' - -[git] -message_template = "Bump to {new_version}" -tag_template = "v{new_version}" - -# For each file to patch, add a [[file]] config -# section containing the path of the file, relative to the -# tbump.toml location. -[[file]] -src = "git_fleximod/cli.py" - -[[file]] -src = "pyproject.toml" - -# You can specify a list of commands to -# run after the files have been patched -# and before the git commit is made - -# [[before_commit]] -# name = "check changelog" -# cmd = "grep -q {new_version} Changelog.rst" - -# Or run some commands after the git tag and the branch -# have been pushed: -# [[after_push]] -# name = "publish" -# cmd = "./publish.sh" diff --git a/.lib/git-fleximod/tests/__init__.py b/.lib/git-fleximod/tests/__init__.py deleted file mode 100644 index 4d4c66c7..00000000 --- a/.lib/git-fleximod/tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -import sys, os - -sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir, "src")) diff --git a/.lib/git-fleximod/tests/conftest.py b/.lib/git-fleximod/tests/conftest.py deleted file mode 100644 index 44d28e17..00000000 --- a/.lib/git-fleximod/tests/conftest.py +++ /dev/null @@ -1,150 +0,0 @@ -import pytest -from git_fleximod.gitinterface import GitInterface -import os -import subprocess -import logging -from pathlib import Path - -@pytest.fixture(scope='session') -def logger(): - logging.basicConfig( - level=logging.INFO, format="%(name)s - %(levelname)s - %(message)s", handlers=[logging.StreamHandler()] - ) - logger = logging.getLogger(__name__) - return logger - -all_repos=[ - {"subrepo_path": "modules/test", - "submodule_name": "test_submodule", - "status1" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", - "status2" : "test_submodule at tag MPIserial_2.4.0", - "status3" : "test_submodule at tag MPIserial_2.4.0", - "status4" : "test_submodule at tag MPIserial_2.4.0", - "gitmodules_content" : """ - [submodule "test_submodule"] - path = modules/test - url = https://github.com/ESMCI/mpi-serial.git - fxtag = MPIserial_2.4.0 - fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = ToplevelRequired -"""}, - {"subrepo_path": "modules/test_optional", - "submodule_name": "test_optional", - "status1" : "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", - "status2" : "test_optional at tag MPIserial_2.4.0", - "status3" : "test_optional not checked out, out of sync at tag MPIserial_2.5.4, expected tag is MPIserial_2.4.0 (optional)", - "status4" : "test_optional at tag MPIserial_2.4.0", - "gitmodules_content": """ - [submodule "test_optional"] - path = modules/test_optional - url = https://github.com/ESMCI/mpi-serial.git - fxtag = MPIserial_2.4.0 - fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = ToplevelOptional -"""}, - {"subrepo_path": "modules/test_alwaysoptional", - "submodule_name": "test_alwaysoptional", - "status1" : "test_alwaysoptional MPIserial_2.3.0 is out of sync with .gitmodules e5cf35c", - "status2" : "test_alwaysoptional at hash e5cf35c", - "status3" : "out of sync at tag MPIserial_2.5.4, expected tag is e5cf35c", - "status4" : "test_alwaysoptional at hash e5cf35c", - "gitmodules_content": """ - [submodule "test_alwaysoptional"] - path = modules/test_alwaysoptional - url = https://github.com/ESMCI/mpi-serial.git - fxtag = e5cf35c - fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = AlwaysOptional -"""}, - {"subrepo_path": "modules/test_sparse", - "submodule_name": "test_sparse", - "status1" : "test_sparse at tag MPIserial_2.5.0", - "status2" : "test_sparse at tag MPIserial_2.5.0", - "status3" : "test_sparse at tag MPIserial_2.5.0", - "status4" : "test_sparse at tag MPIserial_2.5.0", - "gitmodules_content": """ - [submodule "test_sparse"] - path = modules/test_sparse - url = https://github.com/ESMCI/mpi-serial.git - fxtag = MPIserial_2.5.0 - fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = AlwaysRequired - fxsparse = ../.sparse_file_list -"""}, -] -@pytest.fixture(params=all_repos) - -def shared_repos(request): - return request.param - -@pytest.fixture -def get_all_repos(): - return all_repos - -def write_sparse_checkout_file(fp): - sparse_content = """m4 -""" - fp.write_text(sparse_content) - -@pytest.fixture -def test_repo(shared_repos, tmp_path, logger): - subrepo_path = shared_repos["subrepo_path"] - submodule_name = shared_repos["submodule_name"] - test_dir = tmp_path / "testrepo" - test_dir.mkdir() - str_path = str(test_dir) - gitp = GitInterface(str_path, logger) - assert test_dir.joinpath(".git").is_dir() - (test_dir / "modules").mkdir() - if "sparse" in submodule_name: - (test_dir / subrepo_path).mkdir() - # Add the sparse checkout file - write_sparse_checkout_file(test_dir / "modules" / ".sparse_file_list") - gitp.git_operation("add","modules/.sparse_file_list") - else: - gitp = GitInterface(str(test_dir), logger) - gitp.git_operation("submodule", "add", "--depth","1","--name", submodule_name, "https://github.com/ESMCI/mpi-serial.git", subrepo_path) - assert test_dir.joinpath(".gitmodules").is_file() - gitp.git_operation("add",subrepo_path) - gitp.git_operation("commit","-a","-m","\"add submod\"") - test_dir2 = tmp_path / "testrepo2" - gitp.git_operation("clone",test_dir,test_dir2) - return test_dir2 - - -@pytest.fixture -def complex_repo(tmp_path, logger): - test_dir = tmp_path / "testcomplex" - test_dir.mkdir() - str_path = str(test_dir) - gitp = GitInterface(str_path, logger) - gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") - gitp.git_operation("fetch", "origin") - gitp.git_operation("checkout", "v0.0.1") - return test_dir - -@pytest.fixture -def complex_update(tmp_path, logger): - test_dir = tmp_path / "testcomplex" - test_dir.mkdir() - str_path = str(test_dir) - gitp = GitInterface(str_path, logger) - gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") - gitp.git_operation("fetch", "origin") - gitp.git_operation("checkout", "v0.0.2") - - return test_dir - -@pytest.fixture -def git_fleximod(): - def _run_fleximod(path, args, input=None): - cmd = ["git", "fleximod"] + args.split() - result = subprocess.run(cmd, cwd=path, input=input, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True) - if result.returncode: - print(result.stdout) - print(result.stderr) - return result - return _run_fleximod - diff --git a/.lib/git-fleximod/tests/test_a_import.py b/.lib/git-fleximod/tests/test_a_import.py deleted file mode 100644 index d5ca878d..00000000 --- a/.lib/git-fleximod/tests/test_a_import.py +++ /dev/null @@ -1,8 +0,0 @@ -# pylint: disable=unused-import -from git_fleximod import cli -from git_fleximod import utils -from git_fleximod.gitinterface import GitInterface -from git_fleximod.gitmodules import GitModules - -def test_import(): - print("here") diff --git a/.lib/git-fleximod/tests/test_b_update.py b/.lib/git-fleximod/tests/test_b_update.py deleted file mode 100644 index 159f1cfa..00000000 --- a/.lib/git-fleximod/tests/test_b_update.py +++ /dev/null @@ -1,26 +0,0 @@ -import pytest -from pathlib import Path - -def test_basic_checkout(git_fleximod, test_repo, shared_repos): - # Prepare a simple .gitmodules - gm = shared_repos['gitmodules_content'] - file_path = (test_repo / ".gitmodules") - repo_name = shared_repos["submodule_name"] - repo_path = shared_repos["subrepo_path"] - - file_path.write_text(gm) - - # Run the command - result = git_fleximod(test_repo, f"update {repo_name}") - - # Assertions - assert result.returncode == 0 - assert Path(test_repo / repo_path).exists() # Did the submodule directory get created? - if "sparse" in repo_name: - assert Path(test_repo / f"{repo_path}/m4").exists() # Did the submodule sparse directory get created? - assert not Path(test_repo / f"{repo_path}/README").exists() # Did only the submodule sparse directory get created? - - status = git_fleximod(test_repo, f"status {repo_name}") - - assert shared_repos["status2"] in status.stdout - diff --git a/.lib/git-fleximod/tests/test_c_required.py b/.lib/git-fleximod/tests/test_c_required.py deleted file mode 100644 index 2ac66145..00000000 --- a/.lib/git-fleximod/tests/test_c_required.py +++ /dev/null @@ -1,43 +0,0 @@ -import pytest -import re -from pathlib import Path - -def test_required(git_fleximod, test_repo, shared_repos): - file_path = (test_repo / ".gitmodules") - gm = shared_repos["gitmodules_content"] - repo_name = shared_repos["submodule_name"] - if file_path.exists(): - with file_path.open("r") as f: - gitmodules_content = f.read() - # add the entry if it does not exist - if repo_name not in gitmodules_content: - file_path.write_text(gitmodules_content+gm) - # or if it is incomplete - elif gm not in gitmodules_content: - file_path.write_text(gm) - else: - file_path.write_text(gm) - result = git_fleximod(test_repo, "update") - assert result.returncode == 0 - status = git_fleximod(test_repo, f"status {repo_name}") - assert shared_repos["status3"] in status.stdout - status = git_fleximod(test_repo, f"update --optional") - assert result.returncode == 0 - status = git_fleximod(test_repo, f"status {repo_name}") - assert shared_repos["status4"] in status.stdout - status = git_fleximod(test_repo, f"update {repo_name}") - assert result.returncode == 0 - status = git_fleximod(test_repo, f"status {repo_name}") - assert shared_repos["status4"] in status.stdout - - text = file_path.read_text() - new_value = "somethingelse" - pattern = r"(^\s*fxtag\s*=\s*).*$" - replacement = r"\1" + new_value - new_text = re.sub(pattern, replacement, text, flags=re.MULTILINE) - - # Write updated content back to file - file_path.write_text(new_text) - - result = git_fleximod(test_repo, f"update {repo_name}") - assert f'fatal: couldn\'t find remote ref' in result.stderr or 'error: pathspec \'somethingelse\' did not match any file(s) known to git' in result.stderr diff --git a/.lib/git-fleximod/tests/test_d_complex.py b/.lib/git-fleximod/tests/test_d_complex.py deleted file mode 100644 index edde7d81..00000000 --- a/.lib/git-fleximod/tests/test_d_complex.py +++ /dev/null @@ -1,66 +0,0 @@ -import pytest -from pathlib import Path -from git_fleximod.gitinterface import GitInterface - -def test_complex_checkout(git_fleximod, complex_repo, logger): - status = git_fleximod(complex_repo, "status") - assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) - assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout) - assert("Complex not checked out, aligned at tag testtag02" in status.stdout) - assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) - - # This should checkout and update test_submodule and complex_sub - result = git_fleximod(complex_repo, "update") - assert result.returncode == 0 - - status = git_fleximod(complex_repo, "status") - assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag02" in status.stdout) - - # now check the complex_sub - root = (complex_repo / "modules" / "complex") - assert(not (root / "libraries" / "gptl" / ".git").exists()) - assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serial" / ".git").exists()) - assert(not (root / "modules" / "mpi-serial2" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / "m4").exists()) - assert(not (root / "modules" / "mpi-sparse" / "README").exists()) - - # update a single optional submodule - - result = git_fleximod(complex_repo, "update ToplevelOptional") - assert result.returncode == 0 - - status = git_fleximod(complex_repo, "status") - assert("ToplevelOptional at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag02" in status.stdout) - assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) - - # Finally update optional - result = git_fleximod(complex_repo, "update --optional") - assert result.returncode == 0 - - status = git_fleximod(complex_repo, "status") - assert("ToplevelOptional at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag02" in status.stdout) - assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout) - - # now check the complex_sub - root = (complex_repo / "modules" / "complex" ) - assert(not (root / "libraries" / "gptl" / ".git").exists()) - assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serial2" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / "m4").exists()) - assert(not (root / "modules" / "mpi-sparse" / "README").exists()) - - diff --git a/.lib/git-fleximod/tests/test_e_complex_update.py b/.lib/git-fleximod/tests/test_e_complex_update.py deleted file mode 100644 index 0c3ab4c6..00000000 --- a/.lib/git-fleximod/tests/test_e_complex_update.py +++ /dev/null @@ -1,69 +0,0 @@ -import pytest -from pathlib import Path -from git_fleximod.gitinterface import GitInterface - -def test_complex_update(git_fleximod, complex_update, logger): - status = git_fleximod(complex_update, "status") - assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) - assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout) - assert("Complex not checked out, out of sync at tag testtag02, expected tag is testtag3" in status.stdout) - assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) - - # This should checkout and update test_submodule and complex_sub - result = git_fleximod(complex_update, "update") - assert result.returncode == 0 - - status = git_fleximod(complex_update, "status") - assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag3" in status.stdout) - - # now check the complex_sub - root = (complex_update / "modules" / "complex") - assert(not (root / "libraries" / "gptl" / ".git").exists()) - assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serialAR" / ".git").exists()) - assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) - assert(not (root / "modules" / "mpi-serial2" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / "m4").exists()) - assert(not (root / "modules" / "mpi-sparse" / "README").exists()) - - # update a single optional submodule - - result = git_fleximod(complex_update, "update ToplevelOptional") - assert result.returncode == 0 - - status = git_fleximod(complex_update, "status") - assert("ToplevelOptional at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag3" in status.stdout) - assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) - - # Finally update optional - result = git_fleximod(complex_update, "update --optional") - assert result.returncode == 0 - - status = git_fleximod(complex_update, "status") - assert("ToplevelOptional at tag v5.3.2" in status.stdout) - assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) - assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) - assert("Complex at tag testtag3" in status.stdout) - assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout) - - # now check the complex_sub - root = (complex_update / "modules" / "complex" ) - assert(not (root / "libraries" / "gptl" / ".git").exists()) - assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) - assert(not (root / "modules" / "mpi-serial" / ".git").exists()) - assert((root / "modules" / "mpi-serialAR" / ".git").exists()) - assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / ".git").exists()) - assert((root / "modules" / "mpi-serial2" / ".git").exists()) - assert((root / "modules" / "mpi-sparse" / "m4").exists()) - assert(not (root / "modules" / "mpi-sparse" / "README").exists()) - - From 92c3cd25522756a66de038cae54de676c005acc4 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 8 Jan 2026 16:10:08 -0700 Subject: [PATCH 143/150] Squashed '.lib/git-fleximod/' content from commit 5796799 git-subtree-dir: .lib/git-fleximod git-subtree-split: 5796799a6c414bed35bb473fab0e5b6c2257a950 --- .github/workflows/pre-commit | 13 + .github/workflows/pytest.yaml | 79 +++ .pre-commit-config.yaml | 18 + CODE_OF_CONDUCT.md | 107 +++ License | 20 + README.md | 108 +++ doc/Makefile | 20 + doc/conf.py | 26 + doc/index.rst | 24 + doc/make.bat | 35 + escomp_install | 25 + git_fleximod/__init__.py | 0 git_fleximod/cli.py | 168 +++++ git_fleximod/git_fleximod.py | 404 +++++++++++ git_fleximod/gitinterface.py | 181 +++++ git_fleximod/gitmodules.py | 96 +++ git_fleximod/lstripreader.py | 43 ++ git_fleximod/metoflexi.py | 243 +++++++ git_fleximod/submodule.py | 571 ++++++++++++++++ git_fleximod/utils.py | 357 ++++++++++ poetry.lock | 1169 ++++++++++++++++++++++++++++++++ pyproject.toml | 43 ++ tbump.toml | 43 ++ tests/__init__.py | 3 + tests/conftest.py | 187 +++++ tests/test_a_import.py | 9 + tests/test_b_update.py | 103 +++ tests/test_c_required.py | 49 ++ tests/test_d_complex.py | 100 +++ tests/test_e_complex_update.py | 103 +++ tests/utils_for_tests.py | 13 + 31 files changed, 4360 insertions(+) create mode 100644 .github/workflows/pre-commit create mode 100644 .github/workflows/pytest.yaml create mode 100644 .pre-commit-config.yaml create mode 100644 CODE_OF_CONDUCT.md create mode 100644 License create mode 100644 README.md create mode 100644 doc/Makefile create mode 100644 doc/conf.py create mode 100644 doc/index.rst create mode 100644 doc/make.bat create mode 100644 escomp_install create mode 100644 git_fleximod/__init__.py create mode 100644 git_fleximod/cli.py create mode 100755 git_fleximod/git_fleximod.py create mode 100644 git_fleximod/gitinterface.py create mode 100644 git_fleximod/gitmodules.py create mode 100644 git_fleximod/lstripreader.py create mode 100755 git_fleximod/metoflexi.py create mode 100644 git_fleximod/submodule.py create mode 100644 git_fleximod/utils.py create mode 100644 poetry.lock create mode 100644 pyproject.toml create mode 100644 tbump.toml create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_a_import.py create mode 100644 tests/test_b_update.py create mode 100644 tests/test_c_required.py create mode 100644 tests/test_d_complex.py create mode 100644 tests/test_e_complex_update.py create mode 100644 tests/utils_for_tests.py diff --git a/.github/workflows/pre-commit b/.github/workflows/pre-commit new file mode 100644 index 00000000..1a6ad008 --- /dev/null +++ b/.github/workflows/pre-commit @@ -0,0 +1,13 @@ +name: pre-commit +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml new file mode 100644 index 00000000..1d8d10f5 --- /dev/null +++ b/.github/workflows/pytest.yaml @@ -0,0 +1,79 @@ +# Run this job on pushes to `main`, and for pull requests. If you don't specify +# `branches: [main], then this actions runs _twice_ on pull requests, which is +# annoying. + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # If you wanted to use multiple Python versions, you'd have specify a matrix in the job and + # reference the matrixe python version here. + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + # Cache the installation of Poetry itself, e.g. the next step. This prevents the workflow + # from installing Poetry every time, which can be slow. Note the use of the Poetry version + # number in the cache key, and the "-0" suffix: this allows you to invalidate the cache + # manually if/when you want to upgrade Poetry, or if something goes wrong. This could be + # mildly cleaner by using an environment variable, but I don't really care. + - name: cache poetry install + uses: actions/cache@v4 + with: + path: ~/.local + key: poetry-1.8.2 + + # Install Poetry. You could do this manually, or there are several actions that do this. + # `snok/install-poetry` seems to be minimal yet complete, and really just calls out to + # Poetry's default install script, which feels correct. I pin the Poetry version here + # because Poetry does occasionally change APIs between versions and I don't want my + # actions to break if it does. + # + # The key configuration value here is `virtualenvs-in-project: true`: this creates the + # venv as a `.venv` in your testing directory, which allows the next step to easily + # cache it. + - uses: snok/install-poetry@v1 + with: + version: 1.8.2 + virtualenvs-create: true + virtualenvs-in-project: true + + # Cache your dependencies (i.e. all the stuff in your `pyproject.toml`). Note the cache + # key: if you're using multiple Python versions, or multiple OSes, you'd need to include + # them in the cache key. I'm not, so it can be simple and just depend on the poetry.lock. + - name: cache deps + id: cache-deps + uses: actions/cache@v4 + with: + path: .venv + key: pydeps-${{ hashFiles('**/poetry.lock') }} + + # Install dependencies. `--no-root` means "install all dependencies but not the project + # itself", which is what you want to avoid caching _your_ code. The `if` statement + # ensures this only runs on a cache miss. + - run: poetry install --no-interaction --no-root + if: steps.cache-deps.outputs.cache-hit != 'true' + + # Now install _your_ project. This isn't necessary for many types of projects -- particularly + # things like Django apps don't need this. But it's a good idea since it fully-exercises the + # pyproject.toml and makes that if you add things like console-scripts at some point that + # they'll be installed and working. + - run: poetry install --no-interaction + + # And finally run tests. I'm using pytest and all my pytest config is in my `pyproject.toml` + # so this line is super-simple. But it could be as complex as you need. + - run: | + git config --global user.name "${GITHUB_ACTOR}" + git config --global user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" + poetry run pytest --doctest-modules +# - name: Setup tmate session +# if: ${{ failure() }} +# uses: mxschmitt/action-tmate@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..fc6ec61b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +exclude: ^utils/.*$ + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + - repo: https://github.com/PyCQA/pylint + rev: v2.11.1 + hooks: + - id: pylint + args: + - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error,no-member diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..f2c52d5b --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,107 @@ +# Contributor Code of Conduct +_The Contributor Code of Conduct is for participants in our software projects and community._ + +## Our Pledge +We, as contributors, creators, stewards, and maintainers (participants), of **git-fleximod** pledge to make participation in +our software, system or hardware project and community a safe, productive, welcoming and inclusive experience for everyone. +All participants are required to abide by this Code of Conduct. +This includes respectful treatment of everyone regardless of age, body size, disability, ethnicity, gender identity or expression, +level of experience, nationality, political affiliation, veteran status, pregnancy, genetic information, physical appearance, race, +religion, or sexual orientation, as well as any other characteristic protected under applicable US federal or state law. + +## Our Standards +Examples of behaviors that contribute to a positive environment include: + +* All participants are treated with respect and consideration, valuing a diversity of views and opinions +* Be considerate, respectful, and collaborative +* Communicate openly with respect for others, critiquing ideas rather than individuals and gracefully accepting criticism +* Acknowledging the contributions of others +* Avoid personal attacks directed toward other participants +* Be mindful of your surroundings and of your fellow participants +* Alert UCAR staff and suppliers/vendors if you notice a dangerous situation or someone in distress +* Respect the rules and policies of the project and venue + +Examples of unacceptable behavior include, but are not limited to: + +* Harassment, intimidation, or discrimination in any form +* Physical, verbal, or written abuse by anyone to anyone, including repeated use of pronouns other than those requested +* Unwelcome sexual attention or advances +* Personal attacks directed at other guests, members, participants, etc. +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Alarming, intimidating, threatening, or hostile comments or conduct +* Inappropriate use of nudity and/or sexual images +* Threatening or stalking anyone, including a participant +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Scope +This Code of Conduct applies to all spaces managed by the Project whether they be physical, online or face-to-face. +This includes project code, code repository, associated web pages, documentation, mailing lists, project websites and wiki pages, +issue tracker, meetings, telecons, events, project social media accounts, and any other forums created by the project team which the +community uses for communication. +In addition, violations of this Code of Conduct outside these spaces may affect a person's ability to participate within them. +Representation of a project may be further defined and clarified by project maintainers. + +## Community Responsibilities +Everyone in the community is empowered to respond to people who are showing unacceptable behavior. +They can talk to them privately or publicly. +Anyone requested to stop unacceptable behavior is expected to comply immediately. +If the behavior continues concerns may be brought to the project administrators or to any other party listed in the +[Reporting](#reporting) section below. + +## Project Administrator Responsibilities +Project administrators are responsible for clarifying the standards of acceptable behavior and are encouraged to model appropriate +behavior and provide support when people in the community point out inappropriate behavior. +Project administrator(s) are normally the ones that would be tasked to carry out the actions in the [Consequences](#consequences) +section below. + +Project administrators are also expected to keep this Code of Conduct updated with the main one housed at UCAR, as listed below in +the [Attribution](#attribution) section. + +## Reporting +Instances of unacceptable behavior can be brought to the attention of the project administrator(s) who may take any action as +outlined in the [Consequences](#consequences) section below. +However, making a report to a project administrator is not considered an 'official report' to UCAR. + +Instances of unacceptable behavior may also be reported directly to UCAR pursuant to [UCAR's Harassment Reporting and Complaint +Procedure](https://www2.fin.ucar.edu/procedures/hr/harassment-reporting-and-complaint-procedure), or anonymously through [UCAR's +EthicsPoint Hotline](https://www2.fin.ucar.edu/ethics/anonymous-reporting). + +Complaints received by UCAR will be handled pursuant to the procedures outlined in UCAR's Harassment Reporting and Complaint +Procedure. +Complaints to UCAR will be held as confidential as practicable under the circumstances, and retaliation against a person who +initiates a complaint or an inquiry about inappropriate behavior will not be tolerated. + +Any Contributor can use these reporting methods even if they are not directly affiliated with UCAR. +The Frequently Asked Questions (FAQ) page for reporting is [here](https://www2.fin.ucar.edu/procedures/hr/reporting-faqs). + +## Consequences +Upon receipt of a complaint, the project administrator(s) may take any action deemed necessary and appropriate under the +circumstances. +Such action can include things such as: removing, editing, or rejecting comments, commits, code, wiki edits, email, issues, and +other contributions that are not aligned to this Code of Conduct, or banning temporarily or permanently any contributor for other +behaviors that are deemed inappropriate, threatening, offensive, or harmful. +Project administrators also have the right to report violations to UCAR HR and/or UCAR's Office of Diversity, Equity and Inclusion +(ODEI), as well as a participant's home institution and/or law enforcement. +In the event an incident is reported to UCAR, UCAR will follow its Harassment Reporting and Complaint Procedure. + +## Process for Changes +All UCAR managed projects are required to adopt this Contributor Code of Conduct. +Adoption is assumed even if not expressly stated in the repository. +Projects should fill in sections where prompted with project-specific information, including, project name and adoption date. + +Projects that adopt this Code of Conduct need to stay up to date with UCAR's Contributor Code of Conduct, linked with a DOI in the +[Attribution](#attribution) section below. +Projects can make limited substantive changes to the Code of Conduct, however, the changes must be limited in scope and may not +contradict the UCAR Contributor Code of Conduct. + +## Attribution +This Code of Conduct was originally adapted from the [Contributor Covenant](http://contributor-covenant.org/version/1/4), version +1.4. +We then aligned it with the UCAR Participant Code of Conduct, which also borrows from the American Geophysical Union (AGU) Code of +Conduct. +The UCAR Participant Code of Conduct applies to both UCAR employees as well as participants in activities run by UCAR. +The original version of this for all software projects that have strong management from UCAR or UCAR staff is available on the UCAR +website at https://doi.org/10.5065/6w2c-a132. +The date that it was adopted by this project was **Feb/13/2018**. +When responding to complaints, UCAR HR and ODEI will do so based on the latest published version. +Therefore, any project-specific changes should follow the [Process for Changes](#process-for-changes) section above. diff --git a/License b/License new file mode 100644 index 00000000..88bc2251 --- /dev/null +++ b/License @@ -0,0 +1,20 @@ +Copyright 2024 NSF National Center for Atmospheric Sciences (NCAR) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +“Software”), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000..0978a260 --- /dev/null +++ b/README.md @@ -0,0 +1,108 @@ +# git-fleximod + +Flexible, Enhanced Submodule Management for Git + +## Overview + +Git-fleximod is a Python-based tool that extends Git's submodule and sparse checkout capabilities, offering additional features for managing submodules in a more flexible and efficient way. + +## Installation + + If you choose to locate git-fleximod in your path you can access it via command: git fleximod + +## Usage + + Basic Usage: + git fleximod [options] + Available Commands: + status: Display the status of submodules. + update: Update submodules to the tag indicated in .gitmodules variable fxtag. + test: Make sure that fxtags and submodule hashes are consistant, + make sure that official urls (as defined by fxDONOTUSEurl) are set + make sure that fxtags are defined for all submodules + Additional Options: + See git fleximod --help for more details. + +## Supported .gitmodules Variables + + fxtag: Specify a specific tag or hash to checkout for a submodule. Branches are not acceptable. + fxrequired: Mark a submodule's checkout behavior, with allowed values: + - ToplevelRequired: Top-level and required (checked out only when this is the Toplevel module). + - ToplevelOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module). + - AlwaysRequired: Always required (always checked out). + - AlwaysOptional: Always optional (checked out with --optional flag). + fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths. + fxDONOTUSEurl: This is the url used in the test subcommand to assure that protected branches do not point to forks + **NOTE** the fxDONOTUSEurl variable is only used to identify the official project repository and should not be + changed by users. Use the url variable to change to a fork if desired. + +## Sparse Checkouts + + To enable sparse checkout for a submodule, set the fxsparse variable + in the .gitmodules file to the path of a file containing the desired + sparse checkout paths. Git-fleximod will automatically configure + sparse checkout based on this file when applicable commands are run. + See [git-sparse-checkout](https://git-scm.com/docs/git-sparse-checkout#_internalsfull_pattern_set) + for details on the format of this file. + +## Tests + + The git fleximod test action is designed to be used by, for example, github workflows + to assure that protected branches are consistant with respect to submodule hashes and fleximod fxtags + +## Examples + +Here are some common usage examples: + +Update all submodules, including optional ones: +```bash + git fleximod update --optional +``` + +Updating a specific submodule to the fxtag indicated in .gitmodules: + +```bash + git fleximod update submodule-name +``` +Example .gitmodules entry: +```ini, toml + [submodule "cosp2"] + path = src/physics/cosp2/src + url = https://github.com/CFMIP/COSPv2.0 + fxsparse = ../.cosp_sparse_checkout + fxrequired = AlwaysRequired + fxtag = v2.1.4cesm +``` +Explanation: + +This entry indicates that the submodule named cosp2 at tag v2.1.4cesm +should be checked out into the directory src/physics/cosp2/src +relative to the .gitmodules directory. It should be checked out from +the URL https://github.com/CFMIP/COSPv2.0 and use sparse checkout as +described in the file ../.cosp_sparse_checkout relative to the path +directory. It should be checked out anytime this .gitmodules entry is +read. + +Additional example: +```ini, toml + [submodule "cime"] + path = cime + url = https://github.com/jedwards4b/cime + fxrequired = ToplevelRequired + fxtag = cime6.0.198_rme01 +``` + +Explanation: + +This entry indicates that the submodule cime should be checked out +into a directory named cime at tag cime6.0.198_rme01 from the URL +https://github.com/jedwards4b/cime. This should only be done if +the .gitmodules file is at the top level of the repository clone. + +## Contributing + +We welcome contributions! Please see the CONTRIBUTING.md file for guidelines. + +## License + +Git-fleximod is released under the MIT License. diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 00000000..d4bb2cbb --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/doc/conf.py b/doc/conf.py new file mode 100644 index 00000000..423099ee --- /dev/null +++ b/doc/conf.py @@ -0,0 +1,26 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "git-fleximod" +author = "Jim Edwards " +release = "0.4.0" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ["sphinx_argparse_cli"] + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "alabaster" +html_static_path = ["_static"] diff --git a/doc/index.rst b/doc/index.rst new file mode 100644 index 00000000..0f9c1a7f --- /dev/null +++ b/doc/index.rst @@ -0,0 +1,24 @@ +.. git-fleximod documentation master file, created by + sphinx-quickstart on Sat Feb 3 12:02:22 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to git-fleximod's documentation! +======================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: +.. module:: sphinxcontrib.autoprogram +.. sphinx_argparse_cli:: + :module: git_fleximod.cli + :func: get_parser + :prog: git-fleximod + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/doc/make.bat b/doc/make.bat new file mode 100644 index 00000000..32bb2452 --- /dev/null +++ b/doc/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/escomp_install b/escomp_install new file mode 100644 index 00000000..ae782e72 --- /dev/null +++ b/escomp_install @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# updates git-fleximod in an ESCOMP model +# this script should be run from the model root directory, it expects +# git-fleximod to already be installed with the script in bin +# and the classes in lib/python/site-packages +import sys +import shutil +import os + +from glob import iglob + +fleximod_root = sys.argv[1] +fleximod_path = os.path.join(fleximod_root,"src","git-fleximod") +if os.path.isfile(fleximod_path): + with open(fleximod_path,"r") as f: + fleximod = f.readlines() + with open(os.path.join(".","bin","git-fleximod"),"w") as f: + for line in fleximod: + f.write(line) + if "import argparse" in line: + f.write('\nsys.path.append(os.path.join(os.path.dirname(__file__),"..","lib","python","site-packages"))\n\n') + + for file in iglob(os.path.join(fleximod_root, "src", "fleximod", "*.py")): + shutil.copy(file, + os.path.join("lib","python","site-packages","fleximod",os.path.basename(file))) diff --git a/git_fleximod/__init__.py b/git_fleximod/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/git_fleximod/cli.py b/git_fleximod/cli.py new file mode 100644 index 00000000..9551bc96 --- /dev/null +++ b/git_fleximod/cli.py @@ -0,0 +1,168 @@ +from pathlib import Path +import argparse, os, sys + +__version__ = "1.1.1" + + +class CustomArgumentParser(argparse.ArgumentParser): + def print_help(self, file=None): + # First print the default help message + super().print_help(file) + + # Then append the contents of README.md + candidate_paths = [ + Path(sys.prefix) / "share" / "git_fleximod" / "README.md", + Path(__file__).resolve().parent.parent / "README.md", # fallback for dev + ] + for path in candidate_paths: + if os.path.exists(path): + with open(path) as f: + print(f.read(), file=file) + return + print("README.md not found.", file=file) + + +def find_root_dir(filename=".gitmodules"): + """ + Finds the highest directory in tree which contains a file called filename. + + >>> import tempfile, os + >>> cwd = os.getcwd() + >>> with tempfile.TemporaryDirectory() as tmp: + ... subdir = Path(tmp) / 'subdir' + ... subdir.mkdir() + ... f = Path(tmp) / '.gitmodules' + ... _ = f.write_text('') + ... os.chdir(subdir) + ... result = find_root_dir('.gitmodules') == str(tmp) + ... os.chdir(cwd) + ... result + True + """ + d = Path.cwd() + root = Path(d.root) + dirlist = [] + dl = d + while dl != root: + dirlist.append(dl) + dl = dl.parent + dirlist.append(root) + dirlist.reverse() + + for dl in dirlist: + attempt = dl / filename + if attempt.is_file(): + return str(dl) + return None + + +def get_parser(): + description = """ + %(prog)s manages checking out groups of gitsubmodules with additional support for Earth System Models + """ + parser = CustomArgumentParser( + description=description, formatter_class=argparse.RawDescriptionHelpFormatter + ) + + # + # user options + # + choices = ["update", "status", "test"] + parser.add_argument( + "action", + choices=choices, + default="update", + help=f"Subcommand of git-fleximod, choices are {choices[:-1]}", + ) + + parser.add_argument( + "components", + nargs="*", + help="Specific component(s) to checkout. By default, " + "all required submodules are checked out.", + ) + + parser.add_argument( + "-C", + "--path", + default=find_root_dir(), + help="Toplevel repository directory. Defaults to top git directory relative to current.", + ) + + parser.add_argument( + "-g", + "--gitmodules", + nargs="?", + default=".gitmodules", + help="The submodule description filename. " "Default: %(default)s.", + ) + + parser.add_argument( + "-x", + "--exclude", + nargs="*", + help="Component(s) listed in the gitmodules file which should be ignored.", + ) + parser.add_argument( + "-f", + "--force", + action="store_true", + default=False, + help="Override cautions and update or checkout over locally modified repository.", + ) + + parser.add_argument( + "-o", + "--optional", + action="store_true", + default=False, + help="By default only the required submodules " + "are checked out. This flag will also checkout the " + "optional submodules relative to the toplevel directory.", + ) + + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Output additional information to " + "the screen and log file. This flag can be " + "used up to two times, increasing the " + "verbosity level each time.", + ) + + parser.add_argument( + "--no-mods-details", + action="store_true", + default=False, + help="Suppress details on local mods in status output.", + ) + + parser.add_argument( + "-V", + "--version", + action="version", + version=f"%(prog)s {__version__}", + help="Print version and exit.", + ) + + # + # developer options + # + parser.add_argument( + "--backtrace", + action="store_true", + help="DEVELOPER: show exception backtraces as extra " "debugging output", + ) + + parser.add_argument( + "-d", + "--debug", + action="store_true", + default=False, + help="DEVELOPER: output additional debugging " + "information to the screen and log file.", + ) + + return parser diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py new file mode 100755 index 00000000..ce63b8bd --- /dev/null +++ b/git_fleximod/git_fleximod.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python +import sys + +MIN_PYTHON = (3, 7) +if sys.version_info < MIN_PYTHON: + sys.exit("Python %s.%s or later is required." % MIN_PYTHON) + +import os +import shutil +import asyncio +import logging +from configparser import NoOptionError +from git_fleximod import utils +from git_fleximod import cli +from git_fleximod.gitinterface import GitInterface +from git_fleximod.gitmodules import GitModules +from git_fleximod.submodule import Submodule + +logger = logging.getLogger(__name__) + + +def fxrequired_allowed_values(): + return [ + "ToplevelRequired", + "ToplevelOptional", + "AlwaysRequired", + "AlwaysOptional", + "TopLevelRequired", + "TopLevelOptional", + ] + + +def commandline_arguments(args=None): + parser = cli.get_parser() + + if args: + options = parser.parse_args(args) + else: + options = parser.parse_args() + + # explicitly listing a component overrides the optional flag + if options.optional or options.components: + fxrequired = fxrequired_allowed_values() + else: + fxrequired = ["ToplevelRequired", "AlwaysRequired", "TopLevelRequired"] + + action = options.action + if not action: + action = "update" + handlers = [logging.StreamHandler()] + + if options.debug: + try: + open("fleximod.log", "w") + except PermissionError: + sys.exit("ABORT: Could not write file fleximod.log") + level = logging.DEBUG + handlers.append(logging.FileHandler("fleximod.log")) + elif options.verbose: + level = logging.INFO + else: + level = logging.WARNING + # Configure the root logger + logging.basicConfig( + level=level, format="%(name)s - %(levelname)s - %(message)s", handlers=handlers + ) + + if hasattr(options, "version"): + exit() + + return ( + options.path, + options.gitmodules, + fxrequired, + options.components, + options.exclude, + options.force, + options.no_mods_details, + action, + ) + + +def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master"): + """ + This function performs a sparse checkout of a git submodule. It does so by first creating the .git/info/sparse-checkout fileq + in the submodule and then checking out the desired tag. If the submodule is already checked out, it will not be checked out again. + Creating the sparse-checkout file first prevents the entire submodule from being checked out and then removed. This is important + because the submodule may have a large number of files and checking out the entire submodule and then removing it would be time + and disk space consuming. + + Parameters: + root_dir (str): The root directory for the git operation. + name (str): The name of the submodule. + url (str): The URL of the submodule. + path (str): The path to the submodule. + sparsefile (str): The sparse file for the submodule. + tag (str, optional): The tag to checkout. Defaults to "master". + + Returns: + None + """ + logger.info("Called sparse_checkout for {}".format(name)) + rgit = GitInterface(root_dir, logger) + superroot = git_toplevelroot(root_dir, logger) + + if superroot: + gitroot = superroot.strip() + else: + gitroot = root_dir.strip() + assert os.path.isdir(os.path.join(gitroot, ".git")) + # first create the module directory + if not os.path.isdir(os.path.join(root_dir, path)): + os.makedirs(os.path.join(root_dir, path)) + + # initialize a new git repo and set the sparse checkout flag + sprep_repo = os.path.join(root_dir, path) + sprepo_git = GitInterface(sprep_repo, logger) + if os.path.exists(os.path.join(sprep_repo, ".git")): + try: + logger.info("Submodule {} found".format(name)) + chk = sprepo_git.config_get_value("core", "sparseCheckout") + if chk == "true": + logger.info("Sparse submodule {} already checked out".format(name)) + return + except NoOptionError: + logger.debug("Sparse submodule {} not present".format(name)) + except Exception as e: + utils.fatal_error("Unexpected error {} occured.".format(e)) + + sprepo_git.config_set_value("core", "sparseCheckout", "true") + + # set the repository remote + + logger.info("Setting remote origin in {}/{}".format(root_dir, path)) + _, remotelist = sprepo_git.git_operation("remote", "-v") + if url not in remotelist: + sprepo_git.git_operation("remote", "add", "origin", url) + + topgit = os.path.join(gitroot, ".git") + + if gitroot != root_dir and os.path.isfile(os.path.join(root_dir, ".git")): + with open(os.path.join(root_dir, ".git")) as f: + gitpath = os.path.relpath( + os.path.join(root_dir, f.read().split()[1]), + start=os.path.join(root_dir, path), + ) + topgit = os.path.join(gitpath, "modules") + else: + topgit = os.path.relpath( + os.path.join(root_dir, ".git", "modules"), + start=os.path.join(root_dir, path), + ) + + with utils.pushd(sprep_repo): + if not os.path.isdir(topgit): + os.makedirs(topgit) + topgit += os.sep + name + + if os.path.isdir(os.path.join(root_dir, path, ".git")): + with utils.pushd(sprep_repo): + if os.path.isdir(os.path.join(topgit, ".git")): + shutil.rmtree(os.path.join(topgit, ".git")) + shutil.move(".git", topgit) + with open(".git", "w") as f: + f.write("gitdir: " + os.path.relpath(topgit)) + # assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo))) + gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout")) + if os.path.isfile(gitsparse): + logger.warning( + "submodule {} is already initialized {}".format(name, topgit) + ) + return + + with utils.pushd(sprep_repo): + if os.path.isfile(sparsefile): + shutil.copy(sparsefile, gitsparse) + + # Finally checkout the repo + sprepo_git.git_operation("fetch", "origin", "--tags") + sprepo_git.git_operation("checkout", tag) + + print(f"Successfully checked out {name:>20} at {tag}") + rgit.config_set_value(f'submodule "{name}"', "active", "true") + rgit.config_set_value(f'submodule "{name}"', "url", url) + + +def init_submodule_from_gitmodules(gitmodules, name, root_dir, llogger): + path = gitmodules.get(name, "path") + url = gitmodules.get(name, "url") + assert path and url, f"Malformed .gitmodules file {path} {url}" + tag = gitmodules.get(name, "fxtag") + if not tag: + tag = gitmodules.get(name, "hash") + fxurl = gitmodules.get(name, "fxDONOTUSEurl") + fxsparse = gitmodules.get(name, "fxsparse") + fxrequired = gitmodules.get(name, "fxrequired") + return Submodule( + root_dir, + name, + path, + url, + fxtag=tag, + fxurl=fxurl, + fxsparse=fxsparse, + fxrequired=fxrequired, + logger=llogger, + ) + + +def submodules_status( + gitmodules, root_dir, toplevel=False, depth=0, no_mods_details=False +): + testfails = 0 + localmods = 0 + needsupdate = 0 + for name in gitmodules.sections(): + submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) + + result, n, l, t = submod.status(depth=depth, no_mods_details=no_mods_details) + if toplevel or not submod.toplevel(): + print(result) + testfails += t + localmods += l + needsupdate += n + subdir = os.path.join(root_dir, submod.path) + if os.path.exists(os.path.join(subdir, ".gitmodules")): + gsubmod = GitModules(logger, confpath=subdir) + t, l, n = submodules_status( + gsubmod, subdir, depth=depth + 1, no_mods_details=no_mods_details + ) + if toplevel or not submod.toplevel(): + testfails += t + localmods += l + needsupdate += n + + return testfails, localmods, needsupdate + + +def git_toplevelroot(root_dir, llogger): + rgit = GitInterface(root_dir, llogger) + _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + return superroot + + +async def submodules_update(gitmodules, root_dir, requiredlist, force): + async def update_submodule(name, requiredlist, force): + submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) + + if not submod.fxrequired: + submod.fxrequired = "AlwaysRequired" + fxrequired = submod.fxrequired + allowedvalues = fxrequired_allowed_values() + assert fxrequired in allowedvalues + + superroot = git_toplevelroot(root_dir, logger) + + if fxrequired and ( + (superroot and "Toplevel" in fxrequired) or fxrequired not in requiredlist + ): + if "Optional" in fxrequired and "Optional" not in requiredlist: + if fxrequired.startswith("Always"): + print(f"Skipping optional component {name:>20}") + return # continue to next submodule + optional = "AlwaysOptional" in requiredlist + + if fxrequired in requiredlist: + await submod.update() + repodir = os.path.join(root_dir, submod.path) + if os.path.exists(os.path.join(repodir, ".gitmodules")): + # recursively handle this checkout + print(f"Recursively checking out submodules of {name}") + gitsubmodules = GitModules(submod.logger, confpath=repodir) + newrequiredlist = ["AlwaysRequired"] + if optional: + newrequiredlist.append("AlwaysOptional") + await submodules_update( + gitsubmodules, repodir, newrequiredlist, force=force + ) + + tasks = [ + update_submodule(name, requiredlist, force) for name in gitmodules.sections() + ] + await asyncio.gather(*tasks) + + +def local_mods_output(): + text = """\ + The submodules labeled with 'M' above are not in a clean state. + The following are options for how to proceed: + (1) Go into each submodule which is not in a clean state and issue a 'git status' + Either revert or commit your changes so that the submodule is in a clean state. + (2) use the --force option to git-fleximod + (3) you can name the particular submodules to update using the git-fleximod command line + (4) As a last resort you can remove the submodule (via 'rm -fr [directory]') + then rerun git-fleximod update. +""" + print(text) + + +def submodules_test(gitmodules, root_dir, no_mods_details=False): + """ + This function tests the git submodules based on the provided parameters. + + It first checks that fxtags are present and in sync with submodule hashes. + Then it ensures that urls are consistent with fxurls (not forks and not ssh) + and that sparse checkout files exist. + + Parameters: + gitmodules (ConfigParser): The gitmodules configuration. + root_dir (str): The root directory for the git operation. + no_mods_details (bool, optional): If True, suppress details on local mods in status output + + Returns: + int: The number of test failures. + """ + # First check that fxtags are present and in sync with submodule hashes + testfails, localmods, needsupdate = submodules_status( + gitmodules, root_dir, no_mods_details=no_mods_details + ) + print("") + # Then make sure that urls are consistant with fxurls (not forks and not ssh) + # and that sparse checkout files exist + for name in gitmodules.sections(): + url = gitmodules.get(name, "url") + fxurl = gitmodules.get(name, "fxDONOTUSEurl") + fxsparse = gitmodules.get(name, "fxsparse") + path = gitmodules.get(name, "path") + fxurl = fxurl[:-4] if fxurl.endswith(".git") else fxurl + url = url[:-4] if url.endswith(".git") else url + if not fxurl or url.lower() != fxurl.lower(): + print(f"{name:>20} url {url} not in sync with required {fxurl}") + testfails += 1 + if fxsparse and not os.path.isfile(os.path.join(root_dir, path, fxsparse)): + print(f"{name:>20} sparse checkout file {fxsparse} not found") + testfails += 1 + return testfails + localmods + needsupdate + + +def main(): + ( + root_dir, + file_name, + fxrequired, + includelist, + excludelist, + force, + no_mods_details, + action, + ) = commandline_arguments() + # Get a logger for the package + global logger + logger = logging.getLogger(__name__) + + logger.info( + "action is {} root_dir={} file_name={}".format(action, root_dir, file_name) + ) + + if not root_dir or not os.path.isfile(os.path.join(root_dir, file_name)): + if root_dir: + file_path = utils.find_upwards(root_dir, file_name) + + if root_dir is None or file_path is None: + root_dir = "." + utils.fatal_error( + "No {} found in {} or any of it's parents".format(file_name, root_dir) + ) + + root_dir = os.path.dirname(file_path) + logger.info( + "root_dir is {} includelist={} excludelist={}".format( + root_dir, includelist, excludelist + ) + ) + gitmodules = GitModules( + logger, + confpath=root_dir, + conffile=file_name, + includelist=includelist, + excludelist=excludelist, + ) + if not gitmodules.sections(): + sys.exit(f"No submodule components found, root_dir={root_dir}") + retval = 0 + if action == "update": + asyncio.run(submodules_update(gitmodules, root_dir, fxrequired, force)) + elif action == "status": + tfails, lmods, updates = submodules_status( + gitmodules, root_dir, toplevel=True, no_mods_details=no_mods_details + ) + if tfails + lmods + updates > 0: + print( + f" testfails = {tfails}, local mods = {lmods}, needs updates {updates}\n" + ) + if lmods > 0: + local_mods_output() + elif action == "test": + retval = submodules_test(gitmodules, root_dir, no_mods_details=no_mods_details) + else: + utils.fatal_error(f"unrecognized action request {action}") + return retval + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/git_fleximod/gitinterface.py b/git_fleximod/gitinterface.py new file mode 100644 index 00000000..cdd99748 --- /dev/null +++ b/git_fleximod/gitinterface.py @@ -0,0 +1,181 @@ +# git_fleximod/gitinterface.py +import sys +from . import utils +from pathlib import Path +import asyncio + + +class GitInterface: + def __init__(self, repo_path, logger): + if isinstance(repo_path, str): + self.repo_path = Path(repo_path).resolve() + elif isinstance(repo_path, Path): + self.repo_path = repo_path.resolve() + else: + raise TypeError("repo_path must be a str or Path object") + if logger is not None: + self.logger = logger + else: + import logging + + self.logger = logging.getLogger(__name__) + if not self.logger.hasHandlers(): + handler = logging.StreamHandler() + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + handler.setFormatter(formatter) + self.logger.addHandler(handler) + self.logger.setLevel(logging.DEBUG) + self.logger.debug("Initialize GitInterface for {}".format(repo_path)) + try: + import git + + self._use_module = True + try: + self.repo = git.Repo(str(self.repo_path)) # Initialize GitPython repo + except git.exc.InvalidGitRepositoryError: + self.git = git + self._init_git_repo() + msg = "Using GitPython interface to git" + except ImportError: + self._use_module = False + if not (self.repo_path / ".git").exists(): + self._init_git_repo() + msg = "Using shell interface to git" + self.logger.info(msg) + + def _git_command(self, operation, *args): + """ + Internal: Build git command or call GitPython. + >>> gi = GitInterface('.', None) + >>> isinstance(gi._git_command('status'), list) or isinstance(gi._git_command('status'), str) + True + """ + self.logger.info(operation) + if self._use_module and operation != "submodule": + try: + return getattr(self.repo.git, operation)(*args) + except Exception as e: + sys.exit(e) + else: + return ["git", "-C", str(self.repo_path), operation] + list(args) + + def _init_git_repo(self): + if self._use_module: + self.repo = self.git.Repo.init(str(self.repo_path)) + else: + command = ("git", "-C", str(self.repo_path), "init") + utils.execute_subprocess(command) + + def _git_operation_command(self, operation, args): + newargs = [] + for a in args: + # Do not use ssh interface + if isinstance(a, str): + a = a.replace("git@github.com:", "https://github.com/") + newargs.append(a) + + return self._git_command(operation, *newargs) + + # pylint: disable=unused-argument + def git_operation(self, operation, *args, **kwargs): + """ + Run a git operation and return status and output. + >>> gi = GitInterface('.', None) + >>> status, output = gi.git_operation('status') + >>> isinstance(status, int) + True + >>> isinstance(output, str) + True + """ + newargs = [] + for a in args: + # Do not use ssh interface + if isinstance(a, str): + a = a.replace("git@github.com:", "https://github.com/") + newargs.append(a) + + command = self._git_command(operation, *newargs) + if isinstance(command, list): + try: + status, output = utils.execute_subprocess( + command, status_to_caller=True, output_to_caller=True + ) + return status, output.rstrip() + except Exception as e: + sys.exit(e) + else: + return 0, command + + # pylint: disable=unused-argument + async def git_operation_async(self, operation, *args, **kwargs): + """ + Asynchronously run a git operation. + (Doc test not run: async context) + """ + command = self._git_operation_command(operation, args) + if isinstance(command, list): + try: + process = await asyncio.create_subprocess_exec( + *command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await process.communicate() + status = process.returncode + output = stdout.decode().strip() if stdout else stderr.decode().strip() + return status, output + except Exception as e: + sys.exit(e) + else: + return 0, command + + def config_get_value(self, section, name): + """ + Get a git config value. + >>> gi = GitInterface('.', None) + >>> gi.config_get_value('user', 'name') is None or isinstance(gi.config_get_value('user', 'name'), str) + True + """ + if self._use_module: + config = self.repo.config_reader() + try: + val = config.get_value(section, name) + except: + val = None + return val + else: + cmd = ( + "git", + "-C", + str(self.repo_path), + "config", + "--get", + f"{section}.{name}", + ) + output = utils.execute_subprocess(cmd, output_to_caller=True) + return output.strip() + + def config_set_value(self, section, name, value): + """ + Set a git config value. + (Doc test not run: modifies config) + """ + if self._use_module: + with self.repo.config_writer() as writer: + if "." in section: + section = section.replace(".", ' "') + '"' + writer.set_value(section, name, value) + writer.release() # Ensure changes are saved + else: + cmd = ( + "git", + "-C", + str(self.repo_path), + "config", + f"{section}.{name}", + value, + ) + self.logger.info(cmd) + utils.execute_subprocess(cmd, output_to_caller=True) diff --git a/git_fleximod/gitmodules.py b/git_fleximod/gitmodules.py new file mode 100644 index 00000000..7e028489 --- /dev/null +++ b/git_fleximod/gitmodules.py @@ -0,0 +1,96 @@ +# git_fleximod/gitmodules.py +from pathlib import Path +from configparser import RawConfigParser, ConfigParser +from .lstripreader import LstripReader + + +class GitModules(RawConfigParser): + def __init__( + self, + logger, + confpath=Path.cwd(), + conffile=".gitmodules", + includelist=None, + excludelist=None, + ): + """ + confpath: Path to the directory containing the .gitmodules file (defaults to the current working directory). + conffile: Name of the configuration file (defaults to .gitmodules). + includelist: Optional list of submodules to include. + excludelist: Optional list of submodules to exclude. + """ + self.logger = logger + self.logger.debug( + "Creating a GitModules object {} {} {} {}".format( + confpath, conffile, includelist, excludelist + ) + ) + super().__init__() + self.conf_file = Path(confpath) / Path(conffile) + if self.conf_file.exists(): + self.read_file(LstripReader(str(self.conf_file)), source=conffile) + self.includelist = includelist + self.excludelist = excludelist + self.isdirty = False + + def reload(self): + self.clear() + if self.conf_file.exists(): + self.read_file(LstripReader(str(self.conf_file)), source=self.conf_file) + + def set(self, name, option, value): + """ + Sets a configuration value for a specific submodule: + Ensures the appropriate section exists for the submodule. + Calls the parent class's set method to store the value. + """ + self.isdirty = True + self.logger.debug("set called {} {} {}".format(name, option, value)) + section = f'submodule "{name}"' + if not self.has_section(section): + self.add_section(section) + super().set(section, option, str(value)) + + # pylint: disable=redefined-builtin, arguments-differ + def get(self, name, option, raw=False, vars=None, fallback=None): + """ + Retrieves a configuration value for a specific submodule: + Uses the parent class's get method to access the value. + Handles potential errors if the section or option doesn't exist. + """ + self.logger.debug("git get called {} {}".format(name, option)) + section = f'submodule "{name}"' + try: + return ConfigParser.get( + self, section, option, raw=raw, vars=vars, fallback=fallback + ) + except ConfigParser.NoOptionError: + return None + + def save(self): + if self.isdirty: + self.logger.info("Writing {}".format(self.conf_file)) + with open(self.conf_file, "w") as fd: + self.write(fd) + self.isdirty = False + + def __del__(self): + self.save() + + def sections(self): + """Strip the submodule part out of section and just use the name""" + self.logger.debug("calling GitModules sections iterator") + names = [] + for section in ConfigParser.sections(self): + name = section[11:-1] + if self.includelist and name not in self.includelist: + continue + if self.excludelist and name in self.excludelist: + continue + names.append(name) + return names + + def items(self, name, raw=False, vars=None): + self.logger.debug("calling GitModules items for {}".format(name)) + section = f'submodule "{name}"' + return ConfigParser.items(section, raw=raw, vars=vars) diff --git a/git_fleximod/lstripreader.py b/git_fleximod/lstripreader.py new file mode 100644 index 00000000..01d5580e --- /dev/null +++ b/git_fleximod/lstripreader.py @@ -0,0 +1,43 @@ +class LstripReader(object): + "LstripReader formats .gitmodules files to be acceptable for configparser" + + def __init__(self, filename): + with open(filename, "r") as infile: + lines = infile.readlines() + self._lines = list() + self._num_lines = len(lines) + self._index = 0 + for line in lines: + self._lines.append(line.lstrip()) + + def readlines(self): + """Return all the lines from this object's file""" + return self._lines + + def readline(self, size=-1): + """Format and return the next line or raise StopIteration""" + try: + line = self.next() + except StopIteration: + line = "" + + if (size > 0) and (len(line) < size): + return line[0:size] + + return line + + def __iter__(self): + """Begin an iteration""" + self._index = 0 + return self + + def next(self): + """Return the next line or raise StopIteration""" + if self._index >= self._num_lines: + raise StopIteration + + self._index = self._index + 1 + return self._lines[self._index - 1] + + def __next__(self): + return self.next() diff --git a/git_fleximod/metoflexi.py b/git_fleximod/metoflexi.py new file mode 100755 index 00000000..d605b347 --- /dev/null +++ b/git_fleximod/metoflexi.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python +from configparser import ConfigParser +import sys +import shutil +from pathlib import Path +import argparse +import logging +from git_fleximod.gitinterface import GitInterface +from git_fleximod.gitmodules import GitModules +from git_fleximod import utils + +logger = None + + +def find_root_dir(filename=".git"): + d = Path.cwd() + root = Path(d.root) + while d != root: + attempt = d / filename + if attempt.is_dir(): + return d + d = d.parent + return None + + +def get_parser(): + description = """ + %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models + """ + parser = argparse.ArgumentParser( + description=description, formatter_class=argparse.RawDescriptionHelpFormatter + ) + + parser.add_argument( + "-e", + "--externals", + nargs="?", + default="Externals.cfg", + help="The externals description filename. " "Default: %(default)s.", + ) + + parser.add_argument( + "-C", + "--path", + default=find_root_dir(), + help="Toplevel repository directory. Defaults to top git directory relative to current.", + ) + + parser.add_argument( + "-g", + "--gitmodules", + nargs="?", + default=".gitmodules", + help="The submodule description filename. " "Default: %(default)s.", + ) + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Output additional information to " + "the screen and log file. This flag can be " + "used up to two times, increasing the " + "verbosity level each time.", + ) + parser.add_argument( + "-d", + "--debug", + action="store_true", + default=False, + help="DEVELOPER: output additional debugging " + "information to the screen and log file.", + ) + + return parser + + +def commandline_arguments(args=None): + parser = get_parser() + + options = parser.parse_args(args) + handlers = [logging.StreamHandler()] + + if options.debug: + try: + open("fleximod.log", "w") + except PermissionError: + sys.exit("ABORT: Could not write file fleximod.log") + level = logging.DEBUG + handlers.append(logging.FileHandler("fleximod.log")) + elif options.verbose: + level = logging.INFO + else: + level = logging.WARNING + # Configure the root logger + logging.basicConfig( + level=level, format="%(name)s - %(levelname)s - %(message)s", handlers=handlers + ) + + return (options.path, options.gitmodules, options.externals) + + +class ExternalRepoTranslator: + """ + Translates external repositories configured in an INI-style externals file. + """ + + def __init__(self, rootpath, gitmodules, externals): + self.rootpath = rootpath + if gitmodules: + self.gitmodules = GitModules(logger, confpath=rootpath) + self.externals = (rootpath / Path(externals)).resolve() + print(f"Translating {self.externals}") + self.git = GitInterface(rootpath, logger) + + # def __del__(self): + # if (self.rootpath / "save.gitignore"): + + def translate_single_repo( + self, section, tag, url, path, efile, hash_, sparse, protocol + ): + """ + Translates a single repository based on configuration details. + + Args: + rootpath (str): Root path of the main repository. + gitmodules (str): Path to the .gitmodules file. + tag (str): The tag to use for the external repository. + url (str): The URL of the external repository. + path (str): The relative path within the main repository for the external repository. + efile (str): The external file or file containing submodules. + hash_ (str): The commit hash to checkout (if applicable). + sparse (str): Boolean indicating whether to use sparse checkout (if applicable). + protocol (str): The protocol to use (e.g., 'git', 'http'). + """ + assert protocol != "svn", "SVN protocol is not currently supported" + print(f"Translating repository {section}") + if efile: + file_path = Path(path) / Path(efile) + newroot = (self.rootpath / file_path).parent.resolve() + if not newroot.exists(): + newroot.mkdir(parents=True) + logger.info("Newroot is {}".format(newroot)) + newt = ExternalRepoTranslator(newroot, ".gitmodules", efile) + newt.translate_repo() + if protocol == "externals_only": + if tag: + self.gitmodules.set(section, "fxtag", tag) + if hash_: + self.gitmodules.set(section, "fxtag", hash_) + + self.gitmodules.set(section, "fxDONOTUSEurl", url) + if sparse: + self.gitmodules.set(section, "fxsparse", sparse) + self.gitmodules.set(section, "fxrequired", "ToplevelRequired") + else: + newpath = self.rootpath / Path(path) + if newpath.exists(): + shutil.rmtree(newpath) + logger.info("Creating directory {}".format(newpath)) + newpath.mkdir(parents=True) + if tag: + logger.info("cloning {}".format(section)) + try: + self.git.git_operation( + "clone", "-b", tag, "--depth", "1", url, path + ) + except: + self.git.git_operation("clone", url, path) + with utils.pushd(newpath): + ngit = GitInterface(newpath, logger) + ngit.git_operation("checkout", tag) + if hash_: + self.git.git_operation("clone", url, path) + git = GitInterface(newpath, logger) + git.git_operation("fetch", "origin") + git.git_operation("checkout", hash_) + if sparse: + print("setting as sparse submodule {}".format(section)) + sparsefile = newpath / Path(sparse) + newfile = newpath / ".git" / "info" / "sparse-checkout" + print(f"sparsefile {sparsefile} newfile {newfile}") + shutil.copy(sparsefile, newfile) + + logger.info("adding submodule {}".format(section)) + self.gitmodules.save() + self.git.git_operation( + "submodule", "add", "-f", "--name", section, url, path + ) + self.git.git_operation("submodule", "absorbgitdirs") + self.gitmodules.reload() + if tag: + self.gitmodules.set(section, "fxtag", tag) + if hash_: + self.gitmodules.set(section, "fxtag", hash_) + + self.gitmodules.set(section, "fxDONOTUSEurl", url) + if sparse: + self.gitmodules.set(section, "fxsparse", sparse) + self.gitmodules.set(section, "fxrequired", "ToplevelRequired") + + def translate_repo(self): + """ + Translates external repositories defined within an external file. + + Args: + rootpath (str): Root path of the main repository. + gitmodules (str): Path to the .gitmodules file. + external_file (str): The path to the external file containing repository definitions. + """ + econfig = ConfigParser() + econfig.read((self.rootpath / Path(self.externals))) + + for section in econfig.sections(): + if section == "externals_description": + logger.info("skipping section {}".format(section)) + return + logger.info("Translating section {}".format(section)) + tag = econfig.get(section, "tag", raw=False, fallback=None) + url = econfig.get(section, "repo_url", raw=False, fallback=None) + path = econfig.get(section, "local_path", raw=False, fallback=None) + efile = econfig.get(section, "externals", raw=False, fallback=None) + hash_ = econfig.get(section, "hash", raw=False, fallback=None) + sparse = econfig.get(section, "sparse", raw=False, fallback=None) + protocol = econfig.get(section, "protocol", raw=False, fallback=None) + + self.translate_single_repo( + section, tag, url, path, efile, hash_, sparse, protocol + ) + + +def _main(): + rootpath, gitmodules, externals = commandline_arguments() + global logger + logger = logging.getLogger(__name__) + with utils.pushd(rootpath): + t = ExternalRepoTranslator(Path(rootpath), gitmodules, externals) + logger.info("Translating {}".format(rootpath)) + t.translate_repo() + + +if __name__ == "__main__": + sys.exit(_main()) diff --git a/git_fleximod/submodule.py b/git_fleximod/submodule.py new file mode 100644 index 00000000..b786ae94 --- /dev/null +++ b/git_fleximod/submodule.py @@ -0,0 +1,571 @@ +import os +import textwrap +import shutil +import string +from configparser import NoOptionError +from git_fleximod import utils +from git_fleximod.gitinterface import GitInterface + + +class Submodule: + """ + Represents a Git submodule with enhanced features for flexible management. + + Attributes: + name (str): The name of the submodule. + root_dir (str): The root directory of the main project. + path (str): The relative path from the root directory to the submodule. + url (str): The URL of the submodule repository. + fxurl (str): The URL for flexible submodule management (optional). + fxtag (str): The tag for flexible submodule management (optional). + fxsparse (str): Path to the sparse checkout file relative to the submodule path, see git-sparse-checkout for details (optional). + fxrequired (str): Indicates if the submodule is optional or required (optional). + logger (logging.Logger): Logger instance for logging (optional). + """ + + def __init__( + self, + root_dir, + name, + path, + url, + fxtag=None, + fxurl=None, + fxsparse=None, + fxrequired=None, + logger=None, + ): + """ + Initializes a new Submodule instance with the provided attributes. + + >>> sm = Submodule('.', 'foo', 'bar', 'https://example.com') + >>> sm.name + 'foo' + >>> sm.path + 'bar' + >>> sm.url + 'https://example.com' + """ + self.name = name + self.root_dir = root_dir + self.path = path + self.url = url + self.fxurl = fxurl + self.fxtag = fxtag + self.fxsparse = fxsparse + if fxrequired: + self.fxrequired = fxrequired + else: + self.fxrequired = "AlwaysRequired" + self.logger = logger + + def status(self, depth=0, no_mods_details=False): + """ + Checks the status of the submodule and returns 4 parameters: + - result (str): The status of the submodule. + - needsupdate (bool): An indicator if the submodule needs to be updated. + - localmods (bool): An indicator if the submodule has local modifications. + - testfails (bool): An indicator if the submodule has failed a test, this is used for testing purposes. + + Args: + depth (int, optional): depth of this submodule relative to root, used to + indent output for nested submodules + no_mods_details (bool, optional): if True, suppress details on local mods in + status output + """ + + smpath = os.path.join(self.root_dir, self.path) + testfails = False + localmods = False + needsupdate = False + ahash = None + + # The following prefix gives a tree-like output: + tree_prefix_spaces = 3 + if depth == 0: + tree_prefix = "" + else: + tree_prefix = " " * tree_prefix_spaces * (depth - 1) + "└─ " + + full_name = tree_prefix + self.name + name_width = 20 + full_width = name_width + len(tree_prefix) + + optional = "" + if "Optional" in self.fxrequired: + optional = " (optional)" + + if not os.path.exists(os.path.join(smpath, ".git")): + rootgit = GitInterface(self.root_dir, self.logger) + # submodule commands use path, not name + _, tags = rootgit.git_operation("ls-remote", "--tags", self.url) + _, result = rootgit.git_operation("submodule", "status", smpath) + result = result.split() + + if result: + ahash = result[0][1:] + hhash = None + atag = None + for htag in tags.split("\n"): + if htag.endswith("^{}"): + htag = htag[:-3] + if ahash and not atag and ahash in htag: + atag = (htag.split()[1])[10:] + if self.fxtag and not hhash and htag.endswith(self.fxtag): + hhash = htag.split()[0] + if hhash and atag: + break + if self.fxtag and (ahash == hhash or atag == self.fxtag): + result = f"e {full_name:<{full_width}} not checked out, aligned at tag {self.fxtag}{optional}" + needsupdate = True + elif self.fxtag: + _, ahash = rootgit.git_operation( + "submodule", "status", "{}".format(self.path) + ) + ahash = ahash[1 : len(self.fxtag) + 1] + if self.fxtag == ahash: + result = f"e {full_name:<{full_width}} not checked out, aligned at hash {ahash}{optional}" + else: + result = f"e {full_name:<{full_width}} not checked out, out of sync at tag {atag}, expected tag is {self.fxtag}{optional}" + testfails = True + needsupdate = True + else: + result = f"e {full_name:<{full_width}} has no fxtag defined in .gitmodules{optional}" + testfails = False + else: + with utils.pushd(smpath): + git = GitInterface(smpath, self.logger) + _, remote = git.git_operation("remote") + if remote == "": + result = f"e {full_name:<{full_width}} has no associated remote" + testfails = True + needsupdate = True + return result, needsupdate, localmods, testfails + _, rurl = git.git_operation("ls-remote", "--get-url") + _, lines = git.git_operation("log", '--pretty=format:"%h %d"') + line = lines.partition("\n")[0] + parts = line.split() + ahash = parts[0][1:] + atag = None + if len(parts) > 3: + idx = 0 + while idx < len(parts) - 1: + idx = idx + 1 + if parts[idx] == "tag:": + atag = parts[idx + 1] + while ( + atag.endswith(")") + or atag.endswith(",") + or atag.endswith('"') + ): + atag = atag[:-1] + if atag == self.fxtag: + break + + if rurl != self.url: + remote = self._add_remote(git) + git.git_operation("fetch", remote) + + mod_char = " " + _, status_output = git.git_operation( + "status", "--ignore-submodules", "-uno" + ) + if "nothing to commit" not in status_output: + localmods = True + mod_char = "M" + + # Asked for a tag and found that tag + if self.fxtag and atag == self.fxtag: + result = ( + f" {mod_char} {full_name:<{full_width}} at tag {self.fxtag}" + ) + testfails = False + # Asked for and found a hash + elif self.fxtag and ( + ahash[: len(self.fxtag)] == self.fxtag + or (self.fxtag.find(ahash) == 0) + ): + result = f" {mod_char} {full_name:<{full_width}} at hash {ahash}" + + testfails = False + # Asked for and found a hash + elif atag == ahash: + result = f" {mod_char} {full_name:<{full_width}} at hash {ahash}" + + # Did not find requested tag or hash + elif self.fxtag: + result = f"s{mod_char} {full_name:<{full_width}} {atag} {ahash} is out of sync with .gitmodules {self.fxtag}" + testfails = True + needsupdate = True + else: + if atag: + result = f"e{mod_char} {full_name:<{full_width}} has no fxtag defined in .gitmodules, module at {atag}" + else: + result = f"e{mod_char} {full_name:<{full_width}} has no fxtag defined in .gitmodules, module at {ahash}" + testfails = False + + if localmods and not no_mods_details: + # Print details about the local mods, indented below the other + # information about this submodule. + # + # We use a vertical bar to help with the visual alignment of child + # submodules. There are two main goals of the spacing details here: + # 1. If there is a child of this submodule, the vertical bar used here + # should connect with the vertical part of the tree_prefix. + # 2. The details about any local mods should be indented an additional + # 4 spaces beyond the start of the text like "at tag ..." + # + # Here are details on how we accomplish these goals: + # - leading_spaces: This is key for accomplishing the first goal. We + # need 3 spaces for the first three characters in the output (two + # status characters and a space), plus an additional number of + # spaces matching the number of spaces that would be used in the + # tree_prefix of any *child* of this submodule. + # - total_indent: This is the total indent needed to achieve the + # second goal. The first addition of 4 aligns the output with the + # status (e.g., "at tag ..."), accounting for the 3 leading + # characters before the name and the 1 trailing space after the + # name. The second addition of 4 indents the details about local + # mods an additional 4 spaces. + # - trailing_spaces: This gives the correct total indentation given + # that we already have some leading spaces plus a vertical bar + # character. + leading_spaces = " " * (3 + depth * tree_prefix_spaces) + total_indent = full_width + 4 + 4 + trailing_spaces = " " * (total_indent - len(leading_spaces) - 1) + result = ( + result + + "\n" + + textwrap.indent( + status_output, + leading_spaces + "│" + trailing_spaces, + # The following predicate + # makes the vertical bar + # appear even for blank + # lines: + predicate=lambda _: True, + ) + ) + + # print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}") + return result, needsupdate, localmods, testfails + + def _add_remote(self, git): + """ + Adds a new remote to the submodule if it does not already exist. + + This method checks the existing remotes of the submodule. If the submodule's URL is not already listed as a remote, + it attempts to add a new remote. The name for the new remote is generated dynamically to avoid conflicts. If no + remotes exist, it defaults to naming the new remote 'origin'. + + Args: + git (GitInterface): An instance of GitInterface to perform git operations. + + Returns: + str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL. + """ + _, remotes = git.git_operation("remote", "-v") + remotes = remotes.splitlines() + + if remotes: + newremote = "newremote.00" + tmpurl = self.url.replace("git@github.com:", "https://github.com/") + line = next((s for s in remotes if self.url in s or tmpurl in s), None) + if line: + newremote = line.split()[0] + return newremote + else: + i = 0 + while newremote in remotes: + i = i + 1 + newremote = f"newremote.{i:02d}" + else: + newremote = "origin" + git.git_operation("remote", "add", newremote, self.url) + return newremote + + def toplevel(self): + """ + Returns True if the submodule is Toplevel (either Required or Optional) + """ + return True if "Top" in self.fxrequired else False + + def sparse_checkout(self): + """ + Performs a sparse checkout of the submodule. + + This method optimizes the checkout process by only checking out files specified in the submodule's sparse-checkout configuration, + rather than the entire submodule content. It achieves this by first ensuring the `.git/info/sparse-checkout` file is created and + configured in the submodule's directory. Then, it proceeds to checkout the desired tag. If the submodule has already been checked out, + this method will not perform the checkout again. + + This approach is particularly beneficial for submodules with a large number of files, as it significantly reduces the time and disk space + required for the checkout process by avoiding the unnecessary checkout and subsequent removal of unneeded files. + + Returns: + None + """ + self.logger.info("Called sparse_checkout for {}".format(self.name)) + rgit = GitInterface(self.root_dir, self.logger) + status, superroot = rgit.git_operation( + "rev-parse", "--show-superproject-working-tree" + ) + if superroot: + gitroot = superroot.strip() + else: + gitroot = self.root_dir + # Now need to move the .git dir to the submodule location + rootdotgit = os.path.join(self.root_dir, ".git") + while os.path.isfile(rootdotgit): + with open(rootdotgit) as f: + line = f.readline().rstrip() + if line.startswith("gitdir: "): + rootdotgit = os.path.abspath(os.path.join(self.root_dir, line[8:])) + assert os.path.isdir(rootdotgit) + # first create the module directory + if not os.path.isdir(os.path.join(self.root_dir, self.path)): + os.makedirs(os.path.join(self.root_dir, self.path)) + + # initialize a new git repo and set the sparse checkout flag + sprep_repo = os.path.join(self.root_dir, self.path) + sprepo_git = GitInterface(sprep_repo, self.logger) + if os.path.exists(os.path.join(sprep_repo, ".git")): + try: + self.logger.info("Submodule {} found".format(self.name)) + chk = sprepo_git.config_get_value("core", "sparseCheckout") + if chk == "true": + self.logger.info( + "Sparse submodule {} already checked out".format(self.name) + ) + return + except (NoOptionError): + self.logger.debug("Sparse submodule {} not present".format(self.name)) + except Exception as e: + utils.fatal_error("Unexpected error {} occured.".format(e)) + + sprepo_git.config_set_value("core", "sparseCheckout", "true") + + # set the repository remote + + self.logger.info( + "Setting remote origin in {}/{}".format(self.root_dir, self.path) + ) + status, remotes = sprepo_git.git_operation("remote", "-v") + if self.url not in remotes: + sprepo_git.git_operation("remote", "add", "origin", self.url) + + if gitroot != self.root_dir and os.path.isfile( + os.path.join(self.root_dir, ".git") + ): + with open(os.path.join(self.root_dir, ".git")) as f: + gitpath = os.path.relpath( + os.path.join(self.root_dir, f.read().split()[1]), + start=os.path.join(self.root_dir, self.path), + ) + rootdotgit = os.path.join(gitpath, "modules", self.name) + else: + rootdotgit = os.path.relpath( + os.path.join(self.root_dir, ".git", "modules", self.name), + start=os.path.join(self.root_dir, self.path), + ) + + if os.path.isdir(os.path.join(self.root_dir, self.path, ".git")): + with utils.pushd(sprep_repo): + if os.path.isdir(os.path.join(rootdotgit, ".git")): + shutil.rmtree(os.path.join(rootdotgit, ".git")) + shutil.move(".git", rootdotgit) + with open(".git", "w") as f: + f.write("gitdir: " + os.path.relpath(rootdotgit)) + infodir = os.path.join(rootdotgit, "info") + if not os.path.isdir(infodir): + os.makedirs(infodir) + gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout")) + if os.path.isfile(gitsparse): + self.logger.warning( + "submodule {} is already initialized {}".format( + self.name, rootdotgit + ) + ) + os.remove(gitsparse) + + if os.path.isfile(self.fxsparse): + shutil.copy(self.fxsparse, gitsparse) + else: + self.logger.warning( + "submodule {} could not find {}".format( + self.name, self.fxsparse + ) + ) + + # Finally checkout the repo + sprepo_git.git_operation("fetch", "origin", "--tags") + status, _ = sprepo_git.git_operation("checkout", self.fxtag) + if status: + print(f"Error checking out {self.name:>20} at {self.fxtag}") + else: + print(f"Successfully checked out {self.name:>20} at {self.fxtag}") + status, f = sprepo_git.git_operation("status") + # Restore any files deleted from sandbox + for line in f.splitlines(): + if "deleted:" in line: + deleted_file = line.split("deleted:")[1].strip() + sprepo_git.git_operation("checkout", deleted_file) + + rgit.config_set_value("submodule." + self.name, "active", "true") + rgit.config_set_value("submodule." + self.name, "url", self.url) + rgit.config_set_value("submodule." + self.name, "path", self.path) + + async def update(self): + """ + Updates the submodule to the latest or specified version. + + This method handles the update process of the submodule, including checking out the submodule into the specified path, + handling sparse checkouts if configured, and updating the submodule's URL if necessary. It supports both SSH and HTTPS URLs, + automatically converting SSH URLs to HTTPS to avoid issues for users without SSH keys. + + The update process involves the following steps: + 1. If the submodule is configured for sparse checkout, it performs a sparse checkout. + 2. If the submodule is not already checked out, it clones the submodule using the provided URL. + 3. If a specific tag or hash is provided, it checks out that tag; otherwise, it checks out the latest version. + 4. If the root `.git` is a file (indicating a submodule or a worktree), additional steps are taken to integrate the submodule properly. + + Args: + None + Note: + - SSH URLs are automatically converted to HTTPS to accommodate users without SSH keys. + + Returns: + None + """ + git = GitInterface(self.root_dir, self.logger) + repodir = os.path.join(self.root_dir, self.path) + self.logger.info( + "Checkout {} into {}/{}".format(self.name, self.root_dir, self.path) + ) + # if url is provided update to the new url + tag = None + repo_exists = False + if os.path.exists(os.path.join(repodir, ".git")): + self.logger.info("Submodule {} already checked out".format(self.name)) + repo_exists = True + # Look for a .gitmodules file in the newly checkedout repo + if self.fxsparse: + print(f"Sparse checkout {self.name} fxsparse {self.fxsparse}") + if not os.path.isfile(self.fxsparse): + self.logger.info( + "Submodule {} fxsparse file not found".format(self.name) + ) + + self.sparse_checkout() + else: + if not repo_exists and self.url: + # ssh urls cause problems for those who dont have git accounts with ssh keys defined + # but cime has one since e3sm prefers ssh to https, because the .gitmodules file was + # opened with a GitModules object we don't need to worry about restoring the file here + # it will be done by the GitModules class + if self.url.startswith("git@"): + git.git_operation("clone", self.url, self.path) + smgit = GitInterface(repodir, self.logger) + if not tag: + status, tag = smgit.git_operation( + "describe", "--tags", "--always" + ) + smgit.git_operation("checkout", tag) + # Now need to move the .git dir to the submodule location + rootdotgit = os.path.join(self.root_dir, ".git") + if os.path.isfile(rootdotgit): + with open(rootdotgit) as f: + line = f.readline() + if line.startswith("gitdir: "): + rootdotgit = line[8:] + + newpath = os.path.abspath( + os.path.join(self.root_dir, rootdotgit, "modules", self.name) + ) + if os.path.exists(newpath): + shutil.rmtree(os.path.join(repodir, ".git")) + else: + shutil.move(os.path.join(repodir, ".git"), newpath) + + with open(os.path.join(repodir, ".git"), "w") as f: + f.write("gitdir: " + os.path.relpath(newpath, start=repodir)) + + if not os.path.exists(repodir): + parent = os.path.dirname(repodir) + if not os.path.isdir(parent): + os.makedirs(parent) + git.git_operation( + "submodule", "add", "--name", self.name, "--", self.url, self.path + ) + + if not repo_exists: + git.git_operation("submodule", "init", "--", self.path) + await git.git_operation_async("submodule", "update", "--", self.path) + + if self.fxtag: + smgit = GitInterface(repodir, self.logger) + newremote = self._add_remote(smgit) + # Trying to distingush a tag from a hash + allowed = set(string.digits + "abcdef") + status = 0 + if not set(self.fxtag) <= allowed: + # This is a tag + tag = f"refs/tags/{self.fxtag}:refs/tags/{self.fxtag}" + status, _ = smgit.git_operation("fetch", newremote, tag) + else: + # This is likely a hash, so fetch full history just in case + status, _ = smgit.git_operation("fetch", newremote) + if status == 0: + status, _ = smgit.git_operation("checkout", self.fxtag) + if status: + utils.fatal_error( + f"Failed to checkout {self.name} at tag or hash {self.fxtag} from {repodir}" + ) + + if not os.path.exists(os.path.join(repodir, ".git")): + utils.fatal_error( + f"Failed to checkout {self.name} {repo_exists} {repodir} {self.path}" + ) + + if os.path.exists(os.path.join(self.path, ".git")): + submoddir = os.path.join(self.root_dir, self.path) + with utils.pushd(submoddir): + git = GitInterface(submoddir, self.logger) + # first make sure the url is correct + newremote = self._add_remote(git) + status, tags = git.git_operation("tag", "-l") + fxtag = self.fxtag + if fxtag and fxtag not in tags: + git.git_operation("fetch", newremote, "--tags") + status, atag = git.git_operation("describe", "--tags", "--always") + status, files = git.git_operation("diff", "--name-only", "-z") + modfiles = [] + moddirs = [] + if files: + for f in files.split("\0"): + if f: + if os.path.exists(f): + self.logger.info(f"File {f} locally modified") + modfiles.append(f) + elif os.path.isdir(f): + moddirs.append(f) + else: + modfiles.append(f) + if fxtag and fxtag != atag: + try: + status, _ = git.git_operation("checkout", fxtag) + if not status: + print(f"{self.name:>20} updated to {fxtag}") + except Exception as error: + print(error) + + elif not fxtag: + print(f"No fxtag found for submodule {self.name:>20}") + elif modfiles: + print(f"{self.name:>20} has modified files: {modfiles}") + elif moddirs: + print(f"{self.name:>20} has modified directories: {moddirs}") + else: + print(f"{self.name:>20} up to date.") + + return diff --git a/git_fleximod/utils.py b/git_fleximod/utils.py new file mode 100644 index 00000000..57752143 --- /dev/null +++ b/git_fleximod/utils.py @@ -0,0 +1,357 @@ +#!/usr/bin/env python3 +""" +Common public utilities for manic package + +""" + +import logging +import os +import subprocess +import sys +from threading import Timer +from pathlib import Path + +LOCAL_PATH_INDICATOR = "." +# --------------------------------------------------------------------- +# +# functions to massage text for output and other useful utilities +# +# --------------------------------------------------------------------- +from contextlib import contextmanager + + +@contextmanager +def pushd(new_dir): + """context for chdir. usage: with pushd(new_dir)""" + previous_dir = os.getcwd() + os.chdir(new_dir) + try: + yield + finally: + os.chdir(previous_dir) + + +def log_process_output(output): + """Log each line of process output at debug level so it can be + filtered if necessary. By default, output is a single string, and + logging.debug(output) will only put log info heading on the first + line. This makes it hard to filter with grep. + + """ + output = output.split("\n") + for line in output: + logging.debug(line) + + +def printlog(msg, **kwargs): + """Wrapper script around print to ensure that everything printed to + the screen also gets logged. + + """ + logging.info(msg) + if kwargs: + print(msg, **kwargs) + else: + print(msg) + sys.stdout.flush() + + +def find_upwards(root_dir, filename): + """ + Find a file in root dir or any of its parents. + + >>> import tempfile, os + >>> with tempfile.TemporaryDirectory() as tmp: + ... f = os.path.join(tmp, 'testfile.txt') + ... open(f, 'w').close() + ... find_upwards(tmp, 'testfile.txt').name == 'testfile.txt' + True + """ + d = Path(root_dir) + root = Path(d.root) + while d != root: + attempt = d / filename + if attempt.exists(): + return attempt + d = d.parent + return None + + +def last_n_lines(the_string, n_lines, truncation_message=None): + """ + Returns the last n lines of the given string. + """ + + lines = the_string.splitlines(True) + if len(lines) <= n_lines: + return_val = the_string + else: + lines_subset = lines[-n_lines:] + str_truncated = "".join(lines_subset) + if truncation_message: + str_truncated = truncation_message + "\n" + str_truncated + return_val = str_truncated + + return return_val + + +def indent_string(the_string, indent_level): + """ + Indents the given string by a given number of spaces. + + >>> indent_string('a\\nb', 2) + ' a\\n b' + """ + lines = the_string.splitlines(True) + padding = " " * indent_level + lines_indented = [padding + line for line in lines] + return "".join(lines_indented) + + +# --------------------------------------------------------------------- +# +# error handling +# +# --------------------------------------------------------------------- + + +def fatal_error(message): + """ + Error output function + """ + logging.error(message) + raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message)) + + +# --------------------------------------------------------------------- +# +# Data conversion / manipulation +# +# --------------------------------------------------------------------- +def str_to_bool(bool_str): + """Convert a sting representation of as boolean into a true boolean. + + Conversion should be case insensitive. + """ + value = None + str_lower = bool_str.lower() + if str_lower in ("true", "t"): + value = True + elif str_lower in ("false", "f"): + value = False + if value is None: + msg = ( + 'ERROR: invalid boolean string value "{0}". ' + 'Must be "true" or "false"'.format(bool_str) + ) + fatal_error(msg) + return value + + +REMOTE_PREFIXES = ["http://", "https://", "ssh://", "git@"] + + +def is_remote_url(url): + """check if the user provided a local file path instead of a + remote. If so, it must be expanded to an absolute + path. + + """ + remote_url = False + for prefix in REMOTE_PREFIXES: + if url.startswith(prefix): + remote_url = True + return remote_url + + +def split_remote_url(url): + """check if the user provided a local file path or a + remote. If remote, try to strip off protocol info. + + """ + remote_url = is_remote_url(url) + if not remote_url: + return url + + for prefix in REMOTE_PREFIXES: + url = url.replace(prefix, "") + + if "@" in url: + url = url.split("@")[1] + + if ":" in url: + url = url.split(":")[1] + + return url + + +def expand_local_url(url, field): + """check if the user provided a local file path instead of a + remote. If so, it must be expanded to an absolute + path. + + Note: local paths of LOCAL_PATH_INDICATOR have special meaning and + represent local copy only, don't work with the remotes. + + """ + remote_url = is_remote_url(url) + if not remote_url: + if url.strip() == LOCAL_PATH_INDICATOR: + pass + else: + url = os.path.expandvars(url) + url = os.path.expanduser(url) + if not os.path.isabs(url): + msg = ( + 'WARNING: Externals description for "{0}" contains a ' + "url that is not remote and does not expand to an " + "absolute path. Version control operations may " + "fail.\n\nurl={1}".format(field, url) + ) + printlog(msg) + else: + url = os.path.normpath(url) + return url + + +# --------------------------------------------------------------------- +# +# subprocess +# +# --------------------------------------------------------------------- + +# Give the user a helpful message if we detect that a command seems to +# be hanging. +_HANGING_SEC = 300 + + +def _hanging_msg(working_directory, command): + print( + """ + +Command '{command}' +from directory {working_directory} +has taken {hanging_sec} seconds. It may be hanging. + +The command will continue to run, but you may want to abort +git-fleximod with ^C and investigate. A possible cause of hangs is git +requires authentication to access a private repository. On some +systems, git requests for authentication information will not +be displayed to the user. In this case, the program will appear to +hang. Ensure you can run git manually and access all +repositories without entering your authentication information. + +""".format( + command=command, + working_directory=working_directory, + hanging_sec=_HANGING_SEC, + ) + ) + + +def execute_subprocess(commands, status_to_caller=False, output_to_caller=False): + """Wrapper around subprocess.check_output to handle common + exceptions. + + check_output runs a command with arguments and waits + for it to complete. + + check_output raises an exception on a nonzero return code. if + status_to_caller is true, execute_subprocess returns the subprocess + return code, otherwise execute_subprocess treats non-zero return + status as an error and raises an exception. + + """ + cwd = os.getcwd() + msg = "In directory: {0}\nexecute_subprocess running command:".format(cwd) + logging.info(msg) + commands_str = " ".join(str(element) for element in commands) + logging.info(commands_str) + return_to_caller = status_to_caller or output_to_caller + status = -1 + output = "" + hanging_timer = Timer( + _HANGING_SEC, + _hanging_msg, + kwargs={"working_directory": cwd, "command": commands_str}, + ) + hanging_timer.start() + try: + output = subprocess.check_output( + commands, stderr=subprocess.STDOUT, universal_newlines=True + ) + log_process_output(output) + status = 0 + except OSError as error: + msg = failed_command_msg( + "Command execution failed. Does the executable exist?", commands + ) + logging.error(error) + fatal_error(msg) + except ValueError as error: + msg = failed_command_msg( + "DEV_ERROR: Invalid arguments trying to run subprocess", commands + ) + logging.error(error) + fatal_error(msg) + except subprocess.CalledProcessError as error: + # Only report the error if we are NOT returning to the + # caller. If we are returning to the caller, then it may be a + # simple status check. If returning, it is the callers + # responsibility determine if an error occurred and handle it + # appropriately. + msg_context = "Process did not run successfully; " "returned status {0}".format( + error.returncode + ) + msg = failed_command_msg(msg_context, commands, output=error.output) + if not return_to_caller: + logging.error(error) + logging.error(msg) + log_process_output(error.output) + fatal_error(msg) + status = error.returncode + finally: + hanging_timer.cancel() + + if status_to_caller and output_to_caller: + ret_value = (status, output) + elif status_to_caller: + ret_value = status + elif output_to_caller: + ret_value = output + else: + ret_value = None + + return ret_value + + +def failed_command_msg(msg_context, command, output=None): + """Template for consistent error messages from subprocess calls. + + If 'output' is given, it should provide the output from the failed + command + """ + + if output: + output_truncated = last_n_lines( + output, 20, truncation_message="[... Output truncated for brevity ...]" + ) + errmsg = ( + "Failed with output:\n" + indent_string(output_truncated, 4) + "\nERROR: " + ) + else: + errmsg = "" + + command_str = " ".join(command) + errmsg += """In directory + {cwd} +{context}: + {command} +""".format( + cwd=os.getcwd(), context=msg_context, command=command_str + ) + + if output: + errmsg += "See above for output from failed command.\n" + + return errmsg diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 00000000..6c6af17b --- /dev/null +++ b/poetry.lock @@ -0,0 +1,1169 @@ +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. + +[[package]] +name = "alabaster" +version = "0.7.16" +description = "A light, configurable Sphinx theme" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "alabaster-0.7.16-py3-none-any.whl", hash = "sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92"}, + {file = "alabaster-0.7.16.tar.gz", hash = "sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65"}, +] + +[[package]] +name = "babel" +version = "2.17.0" +description = "Internationalization utilities" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2"}, + {file = "babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d"}, +] + +[package.extras] +dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""] + +[[package]] +name = "certifi" +version = "2025.11.12" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "certifi-2025.11.12-py3-none-any.whl", hash = "sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b"}, + {file = "certifi-2025.11.12.tar.gz", hash = "sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316"}, +] + +[[package]] +name = "cfgv" +version = "3.4.0" +description = "Validate configuration and produce human readable error messages." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, + {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.4" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "charset_normalizer-3.4.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-win32.whl", hash = "sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-win_amd64.whl", hash = "sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d"}, + {file = "charset_normalizer-3.4.4-cp310-cp310-win_arm64.whl", hash = "sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-win32.whl", hash = "sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-win_amd64.whl", hash = "sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016"}, + {file = "charset_normalizer-3.4.4-cp311-cp311-win_arm64.whl", hash = "sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-win32.whl", hash = "sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-win_amd64.whl", hash = "sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525"}, + {file = "charset_normalizer-3.4.4-cp312-cp312-win_arm64.whl", hash = "sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-win32.whl", hash = "sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-win_amd64.whl", hash = "sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14"}, + {file = "charset_normalizer-3.4.4-cp313-cp313-win_arm64.whl", hash = "sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-win32.whl", hash = "sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-win_amd64.whl", hash = "sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c"}, + {file = "charset_normalizer-3.4.4-cp314-cp314-win_arm64.whl", hash = "sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-win32.whl", hash = "sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa"}, + {file = "charset_normalizer-3.4.4-cp38-cp38-win_amd64.whl", hash = "sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-win32.whl", hash = "sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-win_amd64.whl", hash = "sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966"}, + {file = "charset_normalizer-3.4.4-cp39-cp39-win_arm64.whl", hash = "sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50"}, + {file = "charset_normalizer-3.4.4-py3-none-any.whl", hash = "sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f"}, + {file = "charset_normalizer-3.4.4.tar.gz", hash = "sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main"] +markers = "sys_platform == \"win32\"" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "distlib" +version = "0.4.0" +description = "Distribution utilities" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16"}, + {file = "distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d"}, +] + +[[package]] +name = "docutils" +version = "0.19" +description = "Docutils -- Python Documentation Utilities" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, + {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version < \"3.11\"" +files = [ + {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, + {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "filelock" +version = "3.19.1" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d"}, + {file = "filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58"}, +] + +[[package]] +name = "fsspec" +version = "2023.12.2" +description = "File-system specification" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "fsspec-2023.12.2-py3-none-any.whl", hash = "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"}, + {file = "fsspec-2023.12.2.tar.gz", hash = "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb"}, +] + +[package.extras] +abfs = ["adlfs"] +adl = ["adlfs"] +arrow = ["pyarrow (>=1)"] +dask = ["dask", "distributed"] +devel = ["pytest", "pytest-cov"] +dropbox = ["dropbox", "dropboxdrivefs", "requests"] +full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"] +fuse = ["fusepy"] +gcs = ["gcsfs"] +git = ["pygit2"] +github = ["requests"] +gs = ["gcsfs"] +gui = ["panel"] +hdfs = ["pyarrow (>=1)"] +http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"] +libarchive = ["libarchive-c"] +oci = ["ocifs"] +s3 = ["s3fs"] +sftp = ["paramiko"] +smb = ["smbprotocol"] +ssh = ["paramiko"] +tqdm = ["tqdm"] + +[[package]] +name = "gitdb" +version = "4.0.12" +description = "Git Object Database" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, + {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, +] + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.45" +description = "GitPython is a Python library used to interact with Git repositories" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "gitpython-3.1.45-py3-none-any.whl", hash = "sha256:8908cb2e02fb3b93b7eb0f2827125cb699869470432cc885f019b8fd0fccff77"}, + {file = "gitpython-3.1.45.tar.gz", hash = "sha256:85b0ee964ceddf211c41b9f27a49086010a190fd8132a24e21f362a4b36a791c"}, +] + +[package.dependencies] +gitdb = ">=4.0.1,<5" +typing-extensions = {version = ">=3.10.0.2", markers = "python_version < \"3.10\""} + +[package.extras] +doc = ["sphinx (>=7.1.2,<7.2)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] + +[[package]] +name = "identify" +version = "2.6.15" +description = "File identification library for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "identify-2.6.15-py2.py3-none-any.whl", hash = "sha256:1181ef7608e00704db228516541eb83a88a9f94433a8c80bb9b5bd54b1d81757"}, + {file = "identify-2.6.15.tar.gz", hash = "sha256:e4f4864b96c6557ef2a1e1c951771838f4edc9df3a72ec7118b338801b11c7bf"}, +] + +[package.extras] +license = ["ukkonen"] + +[[package]] +name = "idna" +version = "3.11" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, + {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, +] + +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + +[[package]] +name = "imagesize" +version = "1.4.1" +description = "Getting image size from png/jpeg/jpeg2000/gif file" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main"] +files = [ + {file = "imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b"}, + {file = "imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"}, +] + +[[package]] +name = "importlib-metadata" +version = "8.7.0" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.9\"" +files = [ + {file = "importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd"}, + {file = "importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000"}, +] + +[package.dependencies] +zipp = ">=3.20" + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +perf = ["ipython"] +test = ["flufl.flake8", "importlib_resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] +type = ["pytest-mypy"] + +[[package]] +name = "iniconfig" +version = "2.1.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, + {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, + {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "markupsafe" +version = "3.0.3" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"}, + {file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"}, + {file = "markupsafe-3.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695"}, + {file = "markupsafe-3.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591"}, + {file = "markupsafe-3.0.3-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c"}, + {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f"}, + {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6"}, + {file = "markupsafe-3.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1"}, + {file = "markupsafe-3.0.3-cp310-cp310-win32.whl", hash = "sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa"}, + {file = "markupsafe-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8"}, + {file = "markupsafe-3.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1"}, + {file = "markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad"}, + {file = "markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a"}, + {file = "markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50"}, + {file = "markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf"}, + {file = "markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f"}, + {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a"}, + {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115"}, + {file = "markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a"}, + {file = "markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19"}, + {file = "markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01"}, + {file = "markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c"}, + {file = "markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e"}, + {file = "markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce"}, + {file = "markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d"}, + {file = "markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d"}, + {file = "markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a"}, + {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b"}, + {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f"}, + {file = "markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b"}, + {file = "markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d"}, + {file = "markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c"}, + {file = "markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f"}, + {file = "markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795"}, + {file = "markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219"}, + {file = "markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6"}, + {file = "markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676"}, + {file = "markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9"}, + {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1"}, + {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc"}, + {file = "markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12"}, + {file = "markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed"}, + {file = "markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5"}, + {file = "markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485"}, + {file = "markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73"}, + {file = "markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37"}, + {file = "markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19"}, + {file = "markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025"}, + {file = "markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6"}, + {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f"}, + {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb"}, + {file = "markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009"}, + {file = "markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354"}, + {file = "markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218"}, + {file = "markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287"}, + {file = "markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe"}, + {file = "markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026"}, + {file = "markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737"}, + {file = "markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97"}, + {file = "markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d"}, + {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda"}, + {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf"}, + {file = "markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe"}, + {file = "markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9"}, + {file = "markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581"}, + {file = "markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4"}, + {file = "markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab"}, + {file = "markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175"}, + {file = "markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634"}, + {file = "markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50"}, + {file = "markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e"}, + {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5"}, + {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523"}, + {file = "markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc"}, + {file = "markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d"}, + {file = "markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9"}, + {file = "markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa"}, + {file = "markupsafe-3.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26"}, + {file = "markupsafe-3.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc"}, + {file = "markupsafe-3.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c"}, + {file = "markupsafe-3.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42"}, + {file = "markupsafe-3.0.3-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b"}, + {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758"}, + {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2"}, + {file = "markupsafe-3.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d"}, + {file = "markupsafe-3.0.3-cp39-cp39-win32.whl", hash = "sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7"}, + {file = "markupsafe-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e"}, + {file = "markupsafe-3.0.3-cp39-cp39-win_arm64.whl", hash = "sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8"}, + {file = "markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698"}, +] + +[[package]] +name = "nodeenv" +version = "1.9.1" +description = "Node.js virtual environment builder" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main"] +files = [ + {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, + {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, +] + +[[package]] +name = "packaging" +version = "25.0" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, + {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, +] + +[[package]] +name = "platformdirs" +version = "4.4.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "platformdirs-4.4.0-py3-none-any.whl", hash = "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85"}, + {file = "platformdirs-4.4.0.tar.gz", hash = "sha256:ca753cf4d81dc309bc67b0ea38fd15dc97bc30ce419a7f58d13eb3bf14c4febf"}, +] + +[package.extras] +docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.1.3)", "sphinx-autodoc-typehints (>=3)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.4)", "pytest-cov (>=6)", "pytest-mock (>=3.14)"] +type = ["mypy (>=1.14.1)"] + +[[package]] +name = "pluggy" +version = "1.6.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, + {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["coverage", "pytest", "pytest-benchmark"] + +[[package]] +name = "pre-commit" +version = "4.3.0" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pre_commit-4.3.0-py2.py3-none-any.whl", hash = "sha256:2b0747ad7e6e967169136edffee14c16e148a778a54e4f967921aa1ebf2308d8"}, + {file = "pre_commit-4.3.0.tar.gz", hash = "sha256:499fe450cc9d42e9d58e606262795ecb64dd05438943c62b66f6a8673da30b16"}, +] + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +virtualenv = ">=20.10.0" + +[[package]] +name = "pyfakefs" +version = "5.10.2" +description = "Implements a fake file system that mocks the Python file system modules." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "pyfakefs-5.10.2-py3-none-any.whl", hash = "sha256:6ff0e84653a71efc6a73f9ee839c3141e3a7cdf4e1fb97666f82ac5b24308d64"}, + {file = "pyfakefs-5.10.2.tar.gz", hash = "sha256:8ae0e5421e08de4e433853a4609a06a1835f4bc2a3ce13b54f36713a897474ba"}, +] + +[[package]] +name = "pygments" +version = "2.19.2" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, + {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pytest" +version = "8.4.2" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79"}, + {file = "pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01"}, +] + +[package.dependencies] +colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1", markers = "python_version < \"3.11\""} +iniconfig = ">=1" +packaging = ">=20" +pluggy = ">=1.5,<2" +pygments = ">=2.7.2" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pyyaml" +version = "6.0.3" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3"}, + {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6"}, + {file = "PyYAML-6.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369"}, + {file = "PyYAML-6.0.3-cp38-cp38-win32.whl", hash = "sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295"}, + {file = "PyYAML-6.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b"}, + {file = "pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b"}, + {file = "pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198"}, + {file = "pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b"}, + {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0"}, + {file = "pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69"}, + {file = "pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e"}, + {file = "pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c"}, + {file = "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e"}, + {file = "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00"}, + {file = "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d"}, + {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a"}, + {file = "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4"}, + {file = "pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b"}, + {file = "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf"}, + {file = "pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196"}, + {file = "pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c"}, + {file = "pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc"}, + {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e"}, + {file = "pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea"}, + {file = "pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5"}, + {file = "pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b"}, + {file = "pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd"}, + {file = "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8"}, + {file = "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5"}, + {file = "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6"}, + {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6"}, + {file = "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be"}, + {file = "pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26"}, + {file = "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c"}, + {file = "pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb"}, + {file = "pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac"}, + {file = "pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788"}, + {file = "pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5"}, + {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764"}, + {file = "pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35"}, + {file = "pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac"}, + {file = "pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3"}, + {file = "pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3"}, + {file = "pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702"}, + {file = "pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c"}, + {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065"}, + {file = "pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65"}, + {file = "pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9"}, + {file = "pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b"}, + {file = "pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da"}, + {file = "pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5"}, + {file = "pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a"}, + {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926"}, + {file = "pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7"}, + {file = "pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0"}, + {file = "pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007"}, + {file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"}, +] + +[[package]] +name = "requests" +version = "2.32.5" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, + {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset_normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "smmap" +version = "5.0.2" +description = "A pure Python implementation of a sliding window memory map manager" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e"}, + {file = "smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5"}, +] + +[[package]] +name = "snowballstemmer" +version = "3.0.1" +description = "This package provides 32 stemmers for 30 languages generated from Snowball algorithms." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*" +groups = ["main"] +files = [ + {file = "snowballstemmer-3.0.1-py3-none-any.whl", hash = "sha256:6cd7b3897da8d6c9ffb968a6781fa6532dce9c3618a4b127d920dab764a19064"}, + {file = "snowballstemmer-3.0.1.tar.gz", hash = "sha256:6d5eeeec8e9f84d4d56b847692bacf79bc2c8e90c7f80ca4444ff8b6f2e52895"}, +] + +[[package]] +name = "sphinx" +version = "5.3.0" +description = "Python documentation generator" +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "Sphinx-5.3.0.tar.gz", hash = "sha256:51026de0a9ff9fc13c05d74913ad66047e104f56a129ff73e174eb5c3ee794b5"}, + {file = "sphinx-5.3.0-py3-none-any.whl", hash = "sha256:060ca5c9f7ba57a08a1219e547b269fadf125ae25b06b9fa7f66768efb652d6d"}, +] + +[package.dependencies] +alabaster = ">=0.7,<0.8" +babel = ">=2.9" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +docutils = ">=0.14,<0.20" +imagesize = ">=1.3" +importlib-metadata = {version = ">=4.8", markers = "python_version < \"3.10\""} +Jinja2 = ">=3.0" +packaging = ">=21.0" +Pygments = ">=2.12" +requests = ">=2.5.0" +snowballstemmer = ">=2.0" +sphinxcontrib-applehelp = "*" +sphinxcontrib-devhelp = "*" +sphinxcontrib-htmlhelp = ">=2.0.0" +sphinxcontrib-jsmath = "*" +sphinxcontrib-qthelp = "*" +sphinxcontrib-serializinghtml = ">=1.1.5" + +[package.extras] +docs = ["sphinxcontrib-websupport"] +lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-bugbear", "flake8-comprehensions", "flake8-simplify", "isort", "mypy (>=0.981)", "sphinx-lint", "types-requests", "types-typed-ast"] +test = ["cython", "html5lib", "pytest (>=4.6)", "typed_ast ; python_version < \"3.8\""] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "2.0.0" +description = "sphinxcontrib-applehelp is a Sphinx extension which outputs Apple help books" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5"}, + {file = "sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1"}, +] + +[package.extras] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] +standalone = ["Sphinx (>=5)"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "2.0.0" +description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp documents" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2"}, + {file = "sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad"}, +] + +[package.extras] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] +standalone = ["Sphinx (>=5)"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.1.0" +description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8"}, + {file = "sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9"}, +] + +[package.extras] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] +standalone = ["Sphinx (>=5)"] +test = ["html5lib", "pytest"] + +[[package]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +description = "A sphinx extension which renders display math in HTML via JavaScript" +optional = false +python-versions = ">=3.5" +groups = ["main"] +files = [ + {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, + {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, +] + +[package.extras] +test = ["flake8", "mypy", "pytest"] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "2.0.0" +description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp documents" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb"}, + {file = "sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab"}, +] + +[package.extras] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] +standalone = ["Sphinx (>=5)"] +test = ["defusedxml (>=0.7.1)", "pytest"] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "2.0.0" +description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331"}, + {file = "sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d"}, +] + +[package.extras] +lint = ["mypy", "ruff (==0.5.5)", "types-docutils"] +standalone = ["Sphinx (>=5)"] +test = ["pytest"] + +[[package]] +name = "tomli" +version = "2.3.0" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version < \"3.11\"" +files = [ + {file = "tomli-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45"}, + {file = "tomli-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba"}, + {file = "tomli-2.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1381caf13ab9f300e30dd8feadb3de072aeb86f1d34a8569453ff32a7dea4bf"}, + {file = "tomli-2.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0e285d2649b78c0d9027570d4da3425bdb49830a6156121360b3f8511ea3441"}, + {file = "tomli-2.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0a154a9ae14bfcf5d8917a59b51ffd5a3ac1fd149b71b47a3a104ca4edcfa845"}, + {file = "tomli-2.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:74bf8464ff93e413514fefd2be591c3b0b23231a77f901db1eb30d6f712fc42c"}, + {file = "tomli-2.3.0-cp311-cp311-win32.whl", hash = "sha256:00b5f5d95bbfc7d12f91ad8c593a1659b6387b43f054104cda404be6bda62456"}, + {file = "tomli-2.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:4dc4ce8483a5d429ab602f111a93a6ab1ed425eae3122032db7e9acf449451be"}, + {file = "tomli-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7d86942e56ded512a594786a5ba0a5e521d02529b3826e7761a05138341a2ac"}, + {file = "tomli-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:73ee0b47d4dad1c5e996e3cd33b8a76a50167ae5f96a2607cbe8cc773506ab22"}, + {file = "tomli-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:792262b94d5d0a466afb5bc63c7daa9d75520110971ee269152083270998316f"}, + {file = "tomli-2.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f195fe57ecceac95a66a75ac24d9d5fbc98ef0962e09b2eddec5d39375aae52"}, + {file = "tomli-2.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e31d432427dcbf4d86958c184b9bfd1e96b5b71f8eb17e6d02531f434fd335b8"}, + {file = "tomli-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b0882799624980785240ab732537fcfc372601015c00f7fc367c55308c186f6"}, + {file = "tomli-2.3.0-cp312-cp312-win32.whl", hash = "sha256:ff72b71b5d10d22ecb084d345fc26f42b5143c5533db5e2eaba7d2d335358876"}, + {file = "tomli-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:1cb4ed918939151a03f33d4242ccd0aa5f11b3547d0cf30f7c74a408a5b99878"}, + {file = "tomli-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5192f562738228945d7b13d4930baffda67b69425a7f0da96d360b0a3888136b"}, + {file = "tomli-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:be71c93a63d738597996be9528f4abe628d1adf5e6eb11607bc8fe1a510b5dae"}, + {file = "tomli-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4665508bcbac83a31ff8ab08f424b665200c0e1e645d2bd9ab3d3e557b6185b"}, + {file = "tomli-2.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4021923f97266babc6ccab9f5068642a0095faa0a51a246a6a02fccbb3514eaf"}, + {file = "tomli-2.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4ea38c40145a357d513bffad0ed869f13c1773716cf71ccaa83b0fa0cc4e42f"}, + {file = "tomli-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ad805ea85eda330dbad64c7ea7a4556259665bdf9d2672f5dccc740eb9d3ca05"}, + {file = "tomli-2.3.0-cp313-cp313-win32.whl", hash = "sha256:97d5eec30149fd3294270e889b4234023f2c69747e555a27bd708828353ab606"}, + {file = "tomli-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0c95ca56fbe89e065c6ead5b593ee64b84a26fca063b5d71a1122bf26e533999"}, + {file = "tomli-2.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cebc6fe843e0733ee827a282aca4999b596241195f43b4cc371d64fc6639da9e"}, + {file = "tomli-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4c2ef0244c75aba9355561272009d934953817c49f47d768070c3c94355c2aa3"}, + {file = "tomli-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c22a8bf253bacc0cf11f35ad9808b6cb75ada2631c2d97c971122583b129afbc"}, + {file = "tomli-2.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0eea8cc5c5e9f89c9b90c4896a8deefc74f518db5927d0e0e8d4a80953d774d0"}, + {file = "tomli-2.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b74a0e59ec5d15127acdabd75ea17726ac4c5178ae51b85bfe39c4f8a278e879"}, + {file = "tomli-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b5870b50c9db823c595983571d1296a6ff3e1b88f734a4c8f6fc6188397de005"}, + {file = "tomli-2.3.0-cp314-cp314-win32.whl", hash = "sha256:feb0dacc61170ed7ab602d3d972a58f14ee3ee60494292d384649a3dc38ef463"}, + {file = "tomli-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:b273fcbd7fc64dc3600c098e39136522650c49bca95df2d11cf3b626422392c8"}, + {file = "tomli-2.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:940d56ee0410fa17ee1f12b817b37a4d4e4dc4d27340863cc67236c74f582e77"}, + {file = "tomli-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f85209946d1fe94416debbb88d00eb92ce9cd5266775424ff81bc959e001acaf"}, + {file = "tomli-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a56212bdcce682e56b0aaf79e869ba5d15a6163f88d5451cbde388d48b13f530"}, + {file = "tomli-2.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5f3ffd1e098dfc032d4d3af5c0ac64f6d286d98bc148698356847b80fa4de1b"}, + {file = "tomli-2.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e01decd096b1530d97d5d85cb4dff4af2d8347bd35686654a004f8dea20fc67"}, + {file = "tomli-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8a35dd0e643bb2610f156cca8db95d213a90015c11fee76c946aa62b7ae7e02f"}, + {file = "tomli-2.3.0-cp314-cp314t-win32.whl", hash = "sha256:a1f7f282fe248311650081faafa5f4732bdbfef5d45fe3f2e702fbc6f2d496e0"}, + {file = "tomli-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:70a251f8d4ba2d9ac2542eecf008b3c8a9fc5c3f9f02c56a9d7952612be2fdba"}, + {file = "tomli-2.3.0-py3-none-any.whl", hash = "sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b"}, + {file = "tomli-2.3.0.tar.gz", hash = "sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549"}, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +description = "Backported and Experimental Type Hints for Python 3.9+" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version < \"3.11\"" +files = [ + {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, + {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, +] + +[[package]] +name = "urllib3" +version = "2.5.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, + {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "virtualenv" +version = "20.35.4" +description = "Virtual Python Environment builder" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "virtualenv-20.35.4-py3-none-any.whl", hash = "sha256:c21c9cede36c9753eeade68ba7d523529f228a403463376cf821eaae2b650f1b"}, + {file = "virtualenv-20.35.4.tar.gz", hash = "sha256:643d3914d73d3eeb0c552cbb12d7e82adf0e504dbf86a3182f8771a153a1971c"}, +] + +[package.dependencies] +distlib = ">=0.3.7,<1" +filelock = ">=3.12.2,<4" +platformdirs = ">=3.9.1,<5" +typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\""} + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8) ; platform_python_implementation == \"PyPy\" or platform_python_implementation == \"GraalVM\" or platform_python_implementation == \"CPython\" and sys_platform == \"win32\" and python_version >= \"3.13\"", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10) ; platform_python_implementation == \"CPython\""] + +[[package]] +name = "wheel" +version = "0.42.0" +description = "A built-package format for Python" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "wheel-0.42.0-py3-none-any.whl", hash = "sha256:177f9c9b0d45c47873b619f5b650346d632cdc35fb5e4d25058e09c9e581433d"}, + {file = "wheel-0.42.0.tar.gz", hash = "sha256:c45be39f7882c9d34243236f2d63cbd58039e360f85d0913425fbd7ceea617a8"}, +] + +[package.extras] +test = ["pytest (>=6.0.0)", "setuptools (>=65)"] + +[[package]] +name = "wrapt" +version = "2.0.1" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "wrapt-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:64b103acdaa53b7caf409e8d45d39a8442fe6dcfec6ba3f3d141e0cc2b5b4dbd"}, + {file = "wrapt-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:91bcc576260a274b169c3098e9a3519fb01f2989f6d3d386ef9cbf8653de1374"}, + {file = "wrapt-2.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ab594f346517010050126fcd822697b25a7031d815bb4fbc238ccbe568216489"}, + {file = "wrapt-2.0.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:36982b26f190f4d737f04a492a68accbfc6fa042c3f42326fdfbb6c5b7a20a31"}, + {file = "wrapt-2.0.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23097ed8bc4c93b7bf36fa2113c6c733c976316ce0ee2c816f64ca06102034ef"}, + {file = "wrapt-2.0.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8bacfe6e001749a3b64db47bcf0341da757c95959f592823a93931a422395013"}, + {file = "wrapt-2.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8ec3303e8a81932171f455f792f8df500fc1a09f20069e5c16bd7049ab4e8e38"}, + {file = "wrapt-2.0.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:3f373a4ab5dbc528a94334f9fe444395b23c2f5332adab9ff4ea82f5a9e33bc1"}, + {file = "wrapt-2.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f49027b0b9503bf6c8cdc297ca55006b80c2f5dd36cecc72c6835ab6e10e8a25"}, + {file = "wrapt-2.0.1-cp310-cp310-win32.whl", hash = "sha256:8330b42d769965e96e01fa14034b28a2a7600fbf7e8f0cc90ebb36d492c993e4"}, + {file = "wrapt-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:1218573502a8235bb8a7ecaed12736213b22dcde9feab115fa2989d42b5ded45"}, + {file = "wrapt-2.0.1-cp310-cp310-win_arm64.whl", hash = "sha256:eda8e4ecd662d48c28bb86be9e837c13e45c58b8300e43ba3c9b4fa9900302f7"}, + {file = "wrapt-2.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e17283f533a0d24d6e5429a7d11f250a58d28b4ae5186f8f47853e3e70d2590"}, + {file = "wrapt-2.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:85df8d92158cb8f3965aecc27cf821461bb5f40b450b03facc5d9f0d4d6ddec6"}, + {file = "wrapt-2.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c1be685ac7700c966b8610ccc63c3187a72e33cab53526a27b2a285a662cd4f7"}, + {file = "wrapt-2.0.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:df0b6d3b95932809c5b3fecc18fda0f1e07452d05e2662a0b35548985f256e28"}, + {file = "wrapt-2.0.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4da7384b0e5d4cae05c97cd6f94faaf78cc8b0f791fc63af43436d98c4ab37bb"}, + {file = "wrapt-2.0.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ec65a78fbd9d6f083a15d7613b2800d5663dbb6bb96003899c834beaa68b242c"}, + {file = "wrapt-2.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7de3cc939be0e1174969f943f3b44e0d79b6f9a82198133a5b7fc6cc92882f16"}, + {file = "wrapt-2.0.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:fb1a5b72cbd751813adc02ef01ada0b0d05d3dcbc32976ce189a1279d80ad4a2"}, + {file = "wrapt-2.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3fa272ca34332581e00bf7773e993d4f632594eb2d1b0b162a9038df0fd971dd"}, + {file = "wrapt-2.0.1-cp311-cp311-win32.whl", hash = "sha256:fc007fdf480c77301ab1afdbb6ab22a5deee8885f3b1ed7afcb7e5e84a0e27be"}, + {file = "wrapt-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:47434236c396d04875180171ee1f3815ca1eada05e24a1ee99546320d54d1d1b"}, + {file = "wrapt-2.0.1-cp311-cp311-win_arm64.whl", hash = "sha256:837e31620e06b16030b1d126ed78e9383815cbac914693f54926d816d35d8edf"}, + {file = "wrapt-2.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1fdbb34da15450f2b1d735a0e969c24bdb8d8924892380126e2a293d9902078c"}, + {file = "wrapt-2.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3d32794fe940b7000f0519904e247f902f0149edbe6316c710a8562fb6738841"}, + {file = "wrapt-2.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:386fb54d9cd903ee0012c09291336469eb7b244f7183d40dc3e86a16a4bace62"}, + {file = "wrapt-2.0.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7b219cb2182f230676308cdcacd428fa837987b89e4b7c5c9025088b8a6c9faf"}, + {file = "wrapt-2.0.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:641e94e789b5f6b4822bb8d8ebbdfc10f4e4eae7756d648b717d980f657a9eb9"}, + {file = "wrapt-2.0.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe21b118b9f58859b5ebaa4b130dee18669df4bd111daad082b7beb8799ad16b"}, + {file = "wrapt-2.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:17fb85fa4abc26a5184d93b3efd2dcc14deb4b09edcdb3535a536ad34f0b4dba"}, + {file = "wrapt-2.0.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:b89ef9223d665ab255ae42cc282d27d69704d94be0deffc8b9d919179a609684"}, + {file = "wrapt-2.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a453257f19c31b31ba593c30d997d6e5be39e3b5ad9148c2af5a7314061c63eb"}, + {file = "wrapt-2.0.1-cp312-cp312-win32.whl", hash = "sha256:3e271346f01e9c8b1130a6a3b0e11908049fe5be2d365a5f402778049147e7e9"}, + {file = "wrapt-2.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:2da620b31a90cdefa9cd0c2b661882329e2e19d1d7b9b920189956b76c564d75"}, + {file = "wrapt-2.0.1-cp312-cp312-win_arm64.whl", hash = "sha256:aea9c7224c302bc8bfc892b908537f56c430802560e827b75ecbde81b604598b"}, + {file = "wrapt-2.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:47b0f8bafe90f7736151f61482c583c86b0693d80f075a58701dd1549b0010a9"}, + {file = "wrapt-2.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:cbeb0971e13b4bd81d34169ed57a6dda017328d1a22b62fda45e1d21dd06148f"}, + {file = "wrapt-2.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb7cffe572ad0a141a7886a1d2efa5bef0bf7fe021deeea76b3ab334d2c38218"}, + {file = "wrapt-2.0.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c8d60527d1ecfc131426b10d93ab5d53e08a09c5fa0175f6b21b3252080c70a9"}, + {file = "wrapt-2.0.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c654eafb01afac55246053d67a4b9a984a3567c3808bb7df2f8de1c1caba2e1c"}, + {file = "wrapt-2.0.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:98d873ed6c8b4ee2418f7afce666751854d6d03e3c0ec2a399bb039cd2ae89db"}, + {file = "wrapt-2.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c9e850f5b7fc67af856ff054c71690d54fa940c3ef74209ad9f935b4f66a0233"}, + {file = "wrapt-2.0.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e505629359cb5f751e16e30cf3f91a1d3ddb4552480c205947da415d597f7ac2"}, + {file = "wrapt-2.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2879af909312d0baf35f08edeea918ee3af7ab57c37fe47cb6a373c9f2749c7b"}, + {file = "wrapt-2.0.1-cp313-cp313-win32.whl", hash = "sha256:d67956c676be5a24102c7407a71f4126d30de2a569a1c7871c9f3cabc94225d7"}, + {file = "wrapt-2.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:9ca66b38dd642bf90c59b6738af8070747b610115a39af2498535f62b5cdc1c3"}, + {file = "wrapt-2.0.1-cp313-cp313-win_arm64.whl", hash = "sha256:5a4939eae35db6b6cec8e7aa0e833dcca0acad8231672c26c2a9ab7a0f8ac9c8"}, + {file = "wrapt-2.0.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a52f93d95c8d38fed0669da2ebdb0b0376e895d84596a976c15a9eb45e3eccb3"}, + {file = "wrapt-2.0.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4e54bbf554ee29fcceee24fa41c4d091398b911da6e7f5d7bffda963c9aed2e1"}, + {file = "wrapt-2.0.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:908f8c6c71557f4deaa280f55d0728c3bca0960e8c3dd5ceeeafb3c19942719d"}, + {file = "wrapt-2.0.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e2f84e9af2060e3904a32cea9bb6db23ce3f91cfd90c6b426757cf7cc01c45c7"}, + {file = "wrapt-2.0.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3612dc06b436968dfb9142c62e5dfa9eb5924f91120b3c8ff501ad878f90eb3"}, + {file = "wrapt-2.0.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6d2d947d266d99a1477cd005b23cbd09465276e302515e122df56bb9511aca1b"}, + {file = "wrapt-2.0.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:7d539241e87b650cbc4c3ac9f32c8d1ac8a54e510f6dca3f6ab60dcfd48c9b10"}, + {file = "wrapt-2.0.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:4811e15d88ee62dbf5c77f2c3ff3932b1e3ac92323ba3912f51fc4016ce81ecf"}, + {file = "wrapt-2.0.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c1c91405fcf1d501fa5d55df21e58ea49e6b879ae829f1039faaf7e5e509b41e"}, + {file = "wrapt-2.0.1-cp313-cp313t-win32.whl", hash = "sha256:e76e3f91f864e89db8b8d2a8311d57df93f01ad6bb1e9b9976d1f2e83e18315c"}, + {file = "wrapt-2.0.1-cp313-cp313t-win_amd64.whl", hash = "sha256:83ce30937f0ba0d28818807b303a412440c4b63e39d3d8fc036a94764b728c92"}, + {file = "wrapt-2.0.1-cp313-cp313t-win_arm64.whl", hash = "sha256:4b55cacc57e1dc2d0991dbe74c6419ffd415fb66474a02335cb10efd1aa3f84f"}, + {file = "wrapt-2.0.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:5e53b428f65ece6d9dad23cb87e64506392b720a0b45076c05354d27a13351a1"}, + {file = "wrapt-2.0.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ad3ee9d0f254851c71780966eb417ef8e72117155cff04821ab9b60549694a55"}, + {file = "wrapt-2.0.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7b822c61ed04ee6ad64bc90d13368ad6eb094db54883b5dde2182f67a7f22c0"}, + {file = "wrapt-2.0.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7164a55f5e83a9a0b031d3ffab4d4e36bbec42e7025db560f225489fa929e509"}, + {file = "wrapt-2.0.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e60690ba71a57424c8d9ff28f8d006b7ad7772c22a4af432188572cd7fa004a1"}, + {file = "wrapt-2.0.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3cd1a4bd9a7a619922a8557e1318232e7269b5fb69d4ba97b04d20450a6bf970"}, + {file = "wrapt-2.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b4c2e3d777e38e913b8ce3a6257af72fb608f86a1df471cb1d4339755d0a807c"}, + {file = "wrapt-2.0.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3d366aa598d69416b5afedf1faa539fac40c1d80a42f6b236c88c73a3c8f2d41"}, + {file = "wrapt-2.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c235095d6d090aa903f1db61f892fffb779c1eaeb2a50e566b52001f7a0f66ed"}, + {file = "wrapt-2.0.1-cp314-cp314-win32.whl", hash = "sha256:bfb5539005259f8127ea9c885bdc231978c06b7a980e63a8a61c8c4c979719d0"}, + {file = "wrapt-2.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:4ae879acc449caa9ed43fc36ba08392b9412ee67941748d31d94e3cedb36628c"}, + {file = "wrapt-2.0.1-cp314-cp314-win_arm64.whl", hash = "sha256:8639b843c9efd84675f1e100ed9e99538ebea7297b62c4b45a7042edb84db03e"}, + {file = "wrapt-2.0.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:9219a1d946a9b32bb23ccae66bdb61e35c62773ce7ca6509ceea70f344656b7b"}, + {file = "wrapt-2.0.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fa4184e74197af3adad3c889a1af95b53bb0466bced92ea99a0c014e48323eec"}, + {file = "wrapt-2.0.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c5ef2f2b8a53b7caee2f797ef166a390fef73979b15778a4a153e4b5fedce8fa"}, + {file = "wrapt-2.0.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e042d653a4745be832d5aa190ff80ee4f02c34b21f4b785745eceacd0907b815"}, + {file = "wrapt-2.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2afa23318136709c4b23d87d543b425c399887b4057936cd20386d5b1422b6fa"}, + {file = "wrapt-2.0.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c72328f668cf4c503ffcf9434c2b71fdd624345ced7941bc6693e61bbe36bef"}, + {file = "wrapt-2.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3793ac154afb0e5b45d1233cb94d354ef7a983708cc3bb12563853b1d8d53747"}, + {file = "wrapt-2.0.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:fec0d993ecba3991645b4857837277469c8cc4c554a7e24d064d1ca291cfb81f"}, + {file = "wrapt-2.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:949520bccc1fa227274da7d03bf238be15389cd94e32e4297b92337df9b7a349"}, + {file = "wrapt-2.0.1-cp314-cp314t-win32.whl", hash = "sha256:be9e84e91d6497ba62594158d3d31ec0486c60055c49179edc51ee43d095f79c"}, + {file = "wrapt-2.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:61c4956171c7434634401db448371277d07032a81cc21c599c22953374781395"}, + {file = "wrapt-2.0.1-cp314-cp314t-win_arm64.whl", hash = "sha256:35cdbd478607036fee40273be8ed54a451f5f23121bd9d4be515158f9498f7ad"}, + {file = "wrapt-2.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:90897ea1cf0679763b62e79657958cd54eae5659f6360fc7d2ccc6f906342183"}, + {file = "wrapt-2.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:50844efc8cdf63b2d90cd3d62d4947a28311e6266ce5235a219d21b195b4ec2c"}, + {file = "wrapt-2.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:49989061a9977a8cbd6d20f2efa813f24bf657c6990a42967019ce779a878dbf"}, + {file = "wrapt-2.0.1-cp38-cp38-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:09c7476ab884b74dce081ad9bfd07fe5822d8600abade571cb1f66d5fc915af6"}, + {file = "wrapt-2.0.1-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1a8a09a004ef100e614beec82862d11fc17d601092c3599afd22b1f36e4137e"}, + {file = "wrapt-2.0.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:89a82053b193837bf93c0f8a57ded6e4b6d88033a499dadff5067e912c2a41e9"}, + {file = "wrapt-2.0.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:f26f8e2ca19564e2e1fdbb6a0e47f36e0efbab1acc31e15471fad88f828c75f6"}, + {file = "wrapt-2.0.1-cp38-cp38-win32.whl", hash = "sha256:115cae4beed3542e37866469a8a1f2b9ec549b4463572b000611e9946b86e6f6"}, + {file = "wrapt-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:c4012a2bd37059d04f8209916aa771dfb564cccb86079072bdcd48a308b6a5c5"}, + {file = "wrapt-2.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:68424221a2dc00d634b54f92441914929c5ffb1c30b3b837343978343a3512a3"}, + {file = "wrapt-2.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6bd1a18f5a797fe740cb3d7a0e853a8ce6461cc62023b630caec80171a6b8097"}, + {file = "wrapt-2.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fb3a86e703868561c5cad155a15c36c716e1ab513b7065bd2ac8ed353c503333"}, + {file = "wrapt-2.0.1-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5dc1b852337c6792aa111ca8becff5bacf576bf4a0255b0f05eb749da6a1643e"}, + {file = "wrapt-2.0.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c046781d422f0830de6329fa4b16796096f28a92c8aef3850674442cdcb87b7f"}, + {file = "wrapt-2.0.1-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f73f9f7a0ebd0db139253d27e5fc8d2866ceaeef19c30ab5d69dcbe35e1a6981"}, + {file = "wrapt-2.0.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:b667189cf8efe008f55bbda321890bef628a67ab4147ebf90d182f2dadc78790"}, + {file = "wrapt-2.0.1-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:a9a83618c4f0757557c077ef71d708ddd9847ed66b7cc63416632af70d3e2308"}, + {file = "wrapt-2.0.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:1e9b121e9aeb15df416c2c960b8255a49d44b4038016ee17af03975992d03931"}, + {file = "wrapt-2.0.1-cp39-cp39-win32.whl", hash = "sha256:1f186e26ea0a55f809f232e92cc8556a0977e00183c3ebda039a807a42be1494"}, + {file = "wrapt-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:bf4cb76f36be5de950ce13e22e7fdf462b35b04665a12b64f3ac5c1bbbcf3728"}, + {file = "wrapt-2.0.1-cp39-cp39-win_arm64.whl", hash = "sha256:d6cc985b9c8b235bd933990cdbf0f891f8e010b65a3911f7a55179cd7b0fc57b"}, + {file = "wrapt-2.0.1-py3-none-any.whl", hash = "sha256:4d2ce1bf1a48c5277d7969259232b57645aae5686dba1eaeade39442277afbca"}, + {file = "wrapt-2.0.1.tar.gz", hash = "sha256:9c9c635e78497cacb81e84f8b11b23e0aacac7a136e73b8e5b2109a1d9fc468f"}, +] + +[package.extras] +dev = ["pytest", "setuptools"] + +[[package]] +name = "zipp" +version = "3.23.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.9\"" +files = [ + {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, + {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] + +[metadata] +lock-version = "2.1" +python-versions = ">=3.9" +content-hash = "dcaa958aee9992b081f43d3efb409a2d8ece8e151ecb99064b741d57a29a86de" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..1e63c773 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[tool.poetry] +name = "git-fleximod" +version = "1.1.1" +description = "Extended support for git-submodule and git-sparse-checkout" +authors = ["Jim Edwards "] +maintainers = ["Jim Edwards "] +license = "MIT" +readme = "README.md" +homepage = "https://github.com/jedwards4b/git-fleximod" +keywords = ["git", "submodule", "sparse-checkout"] +packages = [ +{ include = "git_fleximod"}, +{ include = "doc"}, +{ include = "README.md"}, +] + +[tool.poetry.scripts] +git-fleximod = "git_fleximod.git_fleximod:main" +me2flexi = "git_fleximod.metoflexi:_main" +fsspec = "fsspec.fuse:main" + +[tool.poetry.dependencies] +python = ">=3.9" +GitPython = "^3.1.0" +sphinx = "^5.0.0" +fsspec = "^2023.12.2" +wheel = "^0.42.0" +pytest = "^8.0.0" +pyfakefs = "^5.3.5" +pre-commit = "4.3.0" +wrapt = ">=2.0.0" + +[tool.poetry.urls] +"Bug Tracker" = "https://github.com/jedwards4b/git-fleximod/issues" + +[tool.pytest.ini_options] +markers = [ + "skip_after_first: only run on first iteration" +] + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/tbump.toml b/tbump.toml new file mode 100644 index 00000000..b93fc745 --- /dev/null +++ b/tbump.toml @@ -0,0 +1,43 @@ +# Uncomment this if your project is hosted on GitHub: +github_url = "https://github.com/jedwards4b/git-fleximod/" + +[version] +current = "1.1.1" + +# Example of a semver regexp. +# Make sure this matches current_version before +# using tbump +regex = ''' + (?P\d+) + \. + (?P\d+) + \. + (?P\d+) + ''' + +[git] +message_template = "Bump to {new_version}" +tag_template = "v{new_version}" + +# For each file to patch, add a [[file]] config +# section containing the path of the file, relative to the +# tbump.toml location. +[[file]] +src = "git_fleximod/cli.py" + +[[file]] +src = "pyproject.toml" + +# You can specify a list of commands to +# run after the files have been patched +# and before the git commit is made + +# [[before_commit]] +# name = "check changelog" +# cmd = "grep -q {new_version} Changelog.rst" + +# Or run some commands after the git tag and the branch +# have been pushed: +# [[after_push]] +# name = "publish" +# cmd = "./publish.sh" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..4d4c66c7 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +import sys, os + +sys.path.append(os.path.join(os.path.dirname(__file__), os.path.pardir, "src")) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..9d359fe5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,187 @@ +import pytest +from git_fleximod.gitinterface import GitInterface +import subprocess +import logging + + +@pytest.fixture(scope="session") +def logger(): + logging.basicConfig( + level=logging.INFO, + format="%(name)s - %(levelname)s - %(message)s", + handlers=[logging.StreamHandler()], + ) + llogger = logging.getLogger(__name__) + return llogger + + +all_repos = [ + { + "subrepo_path": "modules/test", + "submodule_name": "test_submodule", + "status1": "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", + "status2": "test_submodule at tag MPIserial_2.4.0", + "status3": "test_submodule at tag MPIserial_2.4.0", + "status4": "test_submodule at tag MPIserial_2.4.0", + "gitmodules_content": """ + [submodule "test_submodule"] + path = modules/test + url = https://github.com/ESMCI/mpi-serial.git + fxtag = MPIserial_2.4.0 + fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = ToplevelRequired +""", + }, + { + "subrepo_path": "modules/test_optional", + "submodule_name": "test_optional", + "status1": "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", + "status2": "test_optional at tag MPIserial_2.4.0", + "status3": "test_optional not checked out, out of sync at tag MPIserial_2.5.4, expected tag is MPIserial_2.4.0 (optional)", + "status4": "test_optional at tag MPIserial_2.4.0", + "gitmodules_content": """ + [submodule "test_optional"] + path = modules/test_optional + url = https://github.com/ESMCI/mpi-serial.git + fxtag = MPIserial_2.4.0 + fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = ToplevelOptional +""", + }, + { + "subrepo_path": "modules/test_alwaysoptional", + "submodule_name": "test_alwaysoptional", + "status1": "test_alwaysoptional MPIserial_2.3.0 is out of sync with .gitmodules e5cf35c", + "status2": "test_alwaysoptional at hash e5cf35c", + "status3": "out of sync at tag MPIserial_2.5.4, expected tag is e5cf35c", + "status4": "test_alwaysoptional at hash e5cf35c", + "gitmodules_content": """ + [submodule "test_alwaysoptional"] + path = modules/test_alwaysoptional + url = https://github.com/ESMCI/mpi-serial.git + fxtag = e5cf35c + fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = AlwaysOptional +""", + }, + { + "subrepo_path": "modules/test_sparse", + "submodule_name": "test_sparse", + "status1": "test_sparse at tag MPIserial_2.5.0", + "status2": "test_sparse at tag MPIserial_2.5.0", + "status3": "test_sparse at tag MPIserial_2.5.0", + "status4": "test_sparse at tag MPIserial_2.5.0", + "gitmodules_content": """ + [submodule "test_sparse"] + path = modules/test_sparse + url = https://github.com/ESMCI/mpi-serial.git + fxtag = MPIserial_2.5.0 + fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = AlwaysRequired + fxsparse = ../.sparse_file_list +""", + }, +] + + +@pytest.fixture(params=all_repos) +def shared_repos(request): + repo_data = request.param + return repo_data + + +@pytest.fixture +def get_all_repos(): + return all_repos + + +def write_sparse_checkout_file(fp): + sparse_content = """m4 +""" + fp.write_text(sparse_content) + + +# pylint: disable=redefined-outer-name +@pytest.fixture +def test_repo(shared_repos, tmp_path, logger): + subrepo_path = shared_repos["subrepo_path"] + submodule_name = shared_repos["submodule_name"] + test_dir = tmp_path / "testrepo" + test_dir.mkdir() + str_path = str(test_dir) + gitp = GitInterface(str_path, logger) + assert test_dir.joinpath(".git").is_dir() + (test_dir / "modules").mkdir() + if "sparse" in submodule_name: + (test_dir / subrepo_path).mkdir() + # Add the sparse checkout file + write_sparse_checkout_file(test_dir / "modules" / ".sparse_file_list") + gitp.git_operation("add", "modules/.sparse_file_list") + else: + gitp = GitInterface(str(test_dir), logger) + gitp.git_operation( + "submodule", + "add", + "--depth", + "1", + "--name", + submodule_name, + "https://github.com/ESMCI/mpi-serial.git", + subrepo_path, + ) + assert test_dir.joinpath(".gitmodules").is_file() + gitp.git_operation("add", subrepo_path) + gitp.git_operation("commit", "-a", "-m", '"add submod"') + test_dir2 = tmp_path / "testrepo2" + gitp.git_operation("clone", test_dir, test_dir2) + return test_dir2 + + +@pytest.fixture +def complex_repo(tmp_path, logger): + test_dir = tmp_path / "testcomplex" + test_dir.mkdir() + str_path = str(test_dir) + gitp = GitInterface(str_path, logger) + gitp.git_operation( + "remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2" + ) + gitp.git_operation("fetch", "origin") + gitp.git_operation("checkout", "v0.0.1") + return test_dir + + +@pytest.fixture +def complex_update(tmp_path, logger): + test_dir = tmp_path / "testcomplex" + test_dir.mkdir() + str_path = str(test_dir) + gitp = GitInterface(str_path, logger) + gitp.git_operation( + "remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2" + ) + gitp.git_operation("fetch", "origin") + gitp.git_operation("checkout", "v0.0.2") + + return test_dir + + +@pytest.fixture +def git_fleximod(): + def _run_fleximod(path, args, user_input=None): + cmd = ["git", "fleximod"] + args.split() + result = subprocess.run( + cmd, + cwd=path, + input=user_input, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + if result.returncode: + print(result.stdout) + print(result.stderr) + return result + + return _run_fleximod diff --git a/tests/test_a_import.py b/tests/test_a_import.py new file mode 100644 index 00000000..939f1436 --- /dev/null +++ b/tests/test_a_import.py @@ -0,0 +1,9 @@ +# pylint: disable=unused-import +from git_fleximod import cli +from git_fleximod import utils +from git_fleximod.gitinterface import GitInterface +from git_fleximod.gitmodules import GitModules + + +def test_import(): + print("here") diff --git a/tests/test_b_update.py b/tests/test_b_update.py new file mode 100644 index 00000000..e19bbb3d --- /dev/null +++ b/tests/test_b_update.py @@ -0,0 +1,103 @@ +# tests/test_b_update.py +import time +from pathlib import Path + + +def test_basic_checkout(git_fleximod, test_repo, shared_repos): + # Prepare a simple .gitmodules + gm = shared_repos["gitmodules_content"] + file_path = test_repo / ".gitmodules" + repo_name = shared_repos["submodule_name"] + repo_path = shared_repos["subrepo_path"] + + file_path.write_text(gm) + + # Run the command + result = git_fleximod(test_repo, f"update {repo_name}") + + # Assertions + assert result.returncode == 0 + assert Path( + test_repo / repo_path + ).exists() # Did the submodule directory get created? + if "sparse" in repo_name: + assert Path( + test_repo / f"{repo_path}/m4" + ).exists() # Did the submodule sparse directory get created? + assert not Path( + test_repo / f"{repo_path}/README" + ).exists() # Did only the submodule sparse directory get created? + + +def test_local_modification_scenarios(git_fleximod, test_repo, shared_repos): + """ + Test three scenarios for local modifications: + 1. Local mods, repo in sync: update leaves local mods alone. + 2. Local mods, repo out-of-sync, no conflict: update brings repo up to date, local mods retained, message shown. + 3. Local mods, repo out-of-sync, conflict: update fails with error. + """ + repo_name = shared_repos["submodule_name"] + repo_path = shared_repos["subrepo_path"] + submodule_dir = test_repo / repo_path + + # Ensure submodule is checked out and at intended tag + gm = shared_repos["gitmodules_content"] + (test_repo / ".gitmodules").write_text(gm) + result = git_fleximod(test_repo, f"update {repo_name}") + assert result.returncode == 0 + assert submodule_dir.exists() + test_file = submodule_dir / "README" + if not test_file.exists(): + # README must exist in the repository for this test. If not, skip this test. + return + + # --- Scenario 1: Local mods, repo in sync --- + original_content = test_file.read_text() + local_mod_content = f"local modification {time.time()}\n" + test_file.write_text(original_content + local_mod_content) + result1 = git_fleximod(test_repo, f"update {repo_name}") + assert result1.returncode == 0 + assert ( + test_file.read_text() == original_content + local_mod_content + ), "Local modification was overwritten when repo was in sync!" + + # --- Scenario 2: Local mods, repo out-of-sync, no conflict --- + # Simulate out-of-sync by checking out previous commit/tag in submodule + import subprocess + + # Try to checkout previous commit (if possible) + log = ( + subprocess.check_output(["git", "log", "--pretty=oneline"], cwd=submodule_dir) + .decode() + .splitlines() + ) + if len(log) > 1: + prev_hash = log[1].split()[0] + subprocess.check_call(["git", "checkout", prev_hash], cwd=submodule_dir) + # Make a local mod that does not conflict + test_file.write_text(original_content + local_mod_content) + result2 = git_fleximod(test_repo, f"update {repo_name}") + assert result2.returncode == 0 + # Should retain local mod and show message + assert ( + test_file.read_text() == original_content + local_mod_content + ), "Local modification was lost after update with no conflict!" + status = git_fleximod(test_repo, f"status {repo_name}") + assert "modified files" in status.stdout or "modified" in status.stdout.lower() + + # --- Scenario 3: Local mods, repo out-of-sync, conflict --- + # Simulate conflict by modifying file and checking out previous commit that changes the same file + if len(log) > 2: + # Recover original README file + subprocess.check_call(["git", "restore", "README"], cwd=submodule_dir) + # Reset to a further previous commit\ + conflict_hash = log[2].split()[0] + subprocess.check_call(["git", "checkout", conflict_hash], cwd=submodule_dir) + # Overwrite file with conflicting content + test_file.write_text("conflicting local mod\n") + try: + git_fleximod(test_repo, f"update {repo_name}") + except Exception as e: + assert "ERROR" in str(e) or "Failed to checkout" in str( + e + ), "Expected error not raised for conflict scenario!" diff --git a/tests/test_c_required.py b/tests/test_c_required.py new file mode 100644 index 00000000..dd3dcf4f --- /dev/null +++ b/tests/test_c_required.py @@ -0,0 +1,49 @@ +# tests/test_c_required.py +import re + +from tests.utils_for_tests import normalize_whitespace + + +def test_required(git_fleximod, test_repo, shared_repos): + file_path = test_repo / ".gitmodules" + gm = shared_repos["gitmodules_content"] + repo_name = shared_repos["submodule_name"] + if file_path.exists(): + with file_path.open("r") as f: + gitmodules_content = f.read() + # add the entry if it does not exist + if repo_name not in gitmodules_content: + file_path.write_text(gitmodules_content + gm) + # or if it is incomplete + elif gm not in gitmodules_content: + file_path.write_text(gm) + else: + file_path.write_text(gm) + result = git_fleximod(test_repo, "update") + assert result.returncode == 0 + status = git_fleximod(test_repo, f"status {repo_name}") + assert shared_repos["status3"] in normalize_whitespace(status.stdout) + status = git_fleximod(test_repo, "update --optional") + assert result.returncode == 0 + status = git_fleximod(test_repo, f"status {repo_name}") + assert shared_repos["status4"] in normalize_whitespace(status.stdout) + status = git_fleximod(test_repo, f"update {repo_name}") + assert result.returncode == 0 + status = git_fleximod(test_repo, f"status {repo_name}") + assert shared_repos["status4"] in normalize_whitespace(status.stdout) + + text = file_path.read_text() + new_value = "somethingelse" + pattern = r"(^\s*fxtag\s*=\s*).*$" + replacement = r"\1" + new_value + new_text = re.sub(pattern, replacement, text, flags=re.MULTILINE) + + # Write updated content back to file + file_path.write_text(new_text) + + result = git_fleximod(test_repo, f"update {repo_name}") + assert ( + "fatal: couldn't find remote ref" in result.stderr + or "error: pathspec 'somethingelse' did not match any file(s) known to git" + in result.stderr + ) diff --git a/tests/test_d_complex.py b/tests/test_d_complex.py new file mode 100644 index 00000000..f7cb9c92 --- /dev/null +++ b/tests/test_d_complex.py @@ -0,0 +1,100 @@ +# tests/test_d_complex.py + +from tests.utils_for_tests import normalize_whitespace + + +def test_complex_checkout(git_fleximod, complex_repo, logger): + status = git_fleximod(complex_repo, "status") + logger.debug("test_complex_checkout status:\n" + status.stdout) + assert ( + "ToplevelOptional not checked out, aligned at tag v5.3.2" + in normalize_whitespace(status.stdout) + ) + assert ( + "ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" + in normalize_whitespace(status.stdout) + ) + assert ( + "AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" + in normalize_whitespace(status.stdout) + ) + assert "Complex not checked out, aligned at tag testtag02" in normalize_whitespace( + status.stdout + ) + assert ( + "AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" + in normalize_whitespace(status.stdout) + ) + + # This should checkout and update test_submodule and complex_sub + result = git_fleximod(complex_repo, "update") + assert result.returncode == 0 + + status = git_fleximod(complex_repo, "status") + assert ( + "ToplevelOptional not checked out, aligned at tag v5.3.2" + in normalize_whitespace(status.stdout) + ) + assert "ToplevelRequired at tag MPIserial_2.5.0" in normalize_whitespace( + status.stdout + ) + assert "AlwaysRequired at tag MPIserial_2.4.0" in normalize_whitespace( + status.stdout + ) + assert "Complex at tag testtag02" in normalize_whitespace(status.stdout) + + # now check the complex_sub + root = complex_repo / "modules" / "complex" + assert not (root / "libraries" / "gptl" / ".git").exists() + assert not (root / "libraries" / "mpi-serial" / ".git").exists() + assert (root / "modules" / "mpi-serial" / ".git").exists() + assert not (root / "modules" / "mpi-serial2" / ".git").exists() + assert (root / "modules" / "mpi-sparse" / ".git").exists() + assert (root / "modules" / "mpi-sparse" / "m4").exists() + assert not (root / "modules" / "mpi-sparse" / "README").exists() + + # update a single optional submodule + + result = git_fleximod(complex_repo, "update ToplevelOptional") + assert result.returncode == 0 + + status = git_fleximod(complex_repo, "status") + assert "ToplevelOptional at tag v5.3.2" in normalize_whitespace(status.stdout) + assert "ToplevelRequired at tag MPIserial_2.5.0" in normalize_whitespace( + status.stdout + ) + assert "AlwaysRequired at tag MPIserial_2.4.0" in normalize_whitespace( + status.stdout + ) + assert "Complex at tag testtag02" in normalize_whitespace(status.stdout) + assert ( + "AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" + in normalize_whitespace(status.stdout) + ) + + # Finally update optional + result = git_fleximod(complex_repo, "update --optional") + assert result.returncode == 0 + + status = git_fleximod(complex_repo, "status") + assert "ToplevelOptional at tag v5.3.2" in normalize_whitespace(status.stdout) + assert "ToplevelRequired at tag MPIserial_2.5.0" in normalize_whitespace( + status.stdout + ) + assert "AlwaysRequired at tag MPIserial_2.4.0" in normalize_whitespace( + status.stdout + ) + assert "Complex at tag testtag02" in normalize_whitespace(status.stdout) + assert "AlwaysOptional at tag MPIserial_2.3.0" in normalize_whitespace( + status.stdout + ) + + # now check the complex_sub + root = complex_repo / "modules" / "complex" + assert not (root / "libraries" / "gptl" / ".git").exists() + assert not (root / "libraries" / "mpi-serial" / ".git").exists() + assert (root / "modules" / "mpi-serial" / ".git").exists() + assert (root / "modules" / "mpi-serial2" / ".git").exists() + assert (root / "modules" / "mpi-sparse" / ".git").exists() + assert (root / "modules" / "mpi-sparse" / "m4").exists() + assert not (root / "modules" / "mpi-sparse" / "README").exists() diff --git a/tests/test_e_complex_update.py b/tests/test_e_complex_update.py new file mode 100644 index 00000000..b529a647 --- /dev/null +++ b/tests/test_e_complex_update.py @@ -0,0 +1,103 @@ +# tests/test_e_complex_update.py +from tests.utils_for_tests import normalize_whitespace + + +def test_complex_update(git_fleximod, complex_update, logger): + logger.debug("Starting test_complex_update") + status = git_fleximod(complex_update, "status") + assert ( + "ToplevelOptional not checked out, aligned at tag v5.3.2" + in normalize_whitespace(status.stdout) + ) + assert ( + "ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" + in normalize_whitespace(status.stdout) + ) + assert ( + "AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" + in normalize_whitespace(status.stdout) + ) + assert ( + "Complex not checked out, out of sync at tag testtag02, expected tag is testtag3" + in normalize_whitespace(status.stdout) + ) + assert ( + "AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" + in normalize_whitespace(status.stdout) + ) + + # This should checkout and update test_submodule and complex_sub + result = git_fleximod(complex_update, "update") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert ( + "ToplevelOptional not checked out, aligned at tag v5.3.2" + in normalize_whitespace(status.stdout) + ) + assert "ToplevelRequired at tag MPIserial_2.5.0" in normalize_whitespace( + status.stdout + ) + assert "AlwaysRequired at tag MPIserial_2.4.0" in normalize_whitespace( + status.stdout + ) + assert "Complex at tag testtag3" in normalize_whitespace(status.stdout) + + # now check the complex_sub + root = complex_update / "modules" / "complex" + assert not (root / "libraries" / "gptl" / ".git").exists() + assert not (root / "libraries" / "mpi-serial" / ".git").exists() + assert (root / "modules" / "mpi-serialAR" / ".git").exists() + assert (root / "modules" / "mpi-serialSAR" / ".git").exists() + assert not (root / "modules" / "mpi-serial2" / ".git").exists() + assert (root / "modules" / "mpi-sparse" / ".git").exists() + assert (root / "modules" / "mpi-sparse" / "m4").exists() + assert not (root / "modules" / "mpi-sparse" / "README").exists() + + # update a single optional submodule + + result = git_fleximod(complex_update, "update ToplevelOptional") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert "ToplevelOptional at tag v5.3.2" in normalize_whitespace(status.stdout) + assert "ToplevelRequired at tag MPIserial_2.5.0" in normalize_whitespace( + status.stdout + ) + assert "AlwaysRequired at tag MPIserial_2.4.0" in normalize_whitespace( + status.stdout + ) + assert "Complex at tag testtag3" in normalize_whitespace(status.stdout) + assert ( + "AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" + in normalize_whitespace(status.stdout) + ) + + # Finally update optional + result = git_fleximod(complex_update, "update --optional") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert "ToplevelOptional at tag v5.3.2" in normalize_whitespace(status.stdout) + assert "ToplevelRequired at tag MPIserial_2.5.0" in normalize_whitespace( + status.stdout + ) + assert "AlwaysRequired at tag MPIserial_2.4.0" in normalize_whitespace( + status.stdout + ) + assert "Complex at tag testtag3" in normalize_whitespace(status.stdout) + assert "AlwaysOptional at tag MPIserial_2.3.0" in normalize_whitespace( + status.stdout + ) + + # now check the complex_sub + root = complex_update / "modules" / "complex" + assert not (root / "libraries" / "gptl" / ".git").exists() + assert not (root / "libraries" / "mpi-serial" / ".git").exists() + assert not (root / "modules" / "mpi-serial" / ".git").exists() + assert (root / "modules" / "mpi-serialAR" / ".git").exists() + assert (root / "modules" / "mpi-serialSAR" / ".git").exists() + assert (root / "modules" / "mpi-sparse" / ".git").exists() + assert (root / "modules" / "mpi-serial2" / ".git").exists() + assert (root / "modules" / "mpi-sparse" / "m4").exists() + assert not (root / "modules" / "mpi-sparse" / "README").exists() diff --git a/tests/utils_for_tests.py b/tests/utils_for_tests.py new file mode 100644 index 00000000..8b6f57e2 --- /dev/null +++ b/tests/utils_for_tests.py @@ -0,0 +1,13 @@ +""" +Helper functions that can be used in tests +""" + + +def normalize_whitespace(text): + """ + Normalize whitespace for flexible string comparisons in tests. + + This removes leading and trailing whitespace and collapses other whitespace down to a + single space. + """ + return " ".join(text.split()) From 8500ae18de354e9fac42cfa076b2a9b5b3fb32ac Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 9 Jan 2026 09:21:29 -0700 Subject: [PATCH 144/150] Remove 'components/cism' directory that is no longer needed. --- components/cism | 1 - 1 file changed, 1 deletion(-) delete mode 160000 components/cism diff --git a/components/cism b/components/cism deleted file mode 160000 index 84767787..00000000 --- a/components/cism +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 84767787b78a3547497cd3f1b452e775cdc56f95 From 532f811917b8be708883d7314cf763b388e47b9e Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 9 Jan 2026 10:18:05 -0700 Subject: [PATCH 145/150] Fix bugs found during CCPP cap/namelist generation. --- .gitmodules | 2 +- cime_config/namelist_definition_cam.xml | 73 ++----------------------- src/physics/ncar_ccpp | 2 +- 3 files changed, 8 insertions(+), 69 deletions(-) diff --git a/.gitmodules b/.gitmodules index 03b58ded..ab0163f4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/nusbaume/atmospheric_physics - fxtag = 40610bd811e1cd34b2a4c2d7d5784a037d3769b6 + fxtag = bcd0a39e94fcfd64e77e0700366dda7b1e04ec22 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/cime_config/namelist_definition_cam.xml b/cime_config/namelist_definition_cam.xml index b2a22e16..0237d591 100644 --- a/cime_config/namelist_definition_cam.xml +++ b/cime_config/namelist_definition_cam.xml @@ -372,69 +372,6 @@ - - - - integer - radiation - radiation_nl - - The frequency at which shortwave radiative - transfer calculations are performed. - positive: time steps; negative: hours - - - -1 - - - - integer - radiation - radiation_nl - - The frequency at which longwave radiative - calculations are performed. - positive: time steps; negative: hours - - - -1 - - - - integer - radiation - radiation_nl - - The amount of time that radiation (both SW and LW) is run continuously from the start of an initial or restart run - positive: time steps; negative: hours - - - -1 - - - - logical - radiation - radiation_nl - - If true, use the namelist-defined, globally-uniform solar zenith angle. - - - .false. - - - - real - kind_phys - radiation - radiation_nl - - The value of the uniform solar zenith angle (in radians) if 'use_rad_uniform_angle' is true. - - - -99 - - char*256 @@ -455,7 +392,8 @@ radiation radiation_nl - The frequency at which shortwave calculation is performed. + The frequency at which shortwave radiative + transfer calculations are performed. positive: time steps; negative: hours @@ -467,7 +405,8 @@ radiation radiation_nl - The frequency at which longwave calculation is performed. + The frequency at which longwave radiative + calculations are performed. positive: time steps; negative: hours @@ -491,7 +430,7 @@ radiation radiation_nl - If true, use the namelist-defined radiation uniform angle in the solar zenith angle calculation + If true, use the namelist-defined, globally-uniform solar zenith angle. .false. @@ -503,7 +442,7 @@ radiation radiation_nl - The radiation timestep size for the solar zenith angle calculation + The value of the uniform solar zenith angle (in radians) if 'use_rad_uniform_angle' is true. -99 diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 40610bd8..bcd0a39e 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 40610bd811e1cd34b2a4c2d7d5784a037d3769b6 +Subproject commit bcd0a39e94fcfd64e77e0700366dda7b1e04ec22 From 6894fa622a6ca91a760732d84be700ee0cefedfc Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 9 Jan 2026 14:36:06 -0700 Subject: [PATCH 146/150] Update atmospheric_physics hash to avoid CAM7 test failure. --- .gitmodules | 2 +- src/physics/ncar_ccpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index ab0163f4..309b91cd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/nusbaume/atmospheric_physics - fxtag = bcd0a39e94fcfd64e77e0700366dda7b1e04ec22 + fxtag = fa881ae5dac60ab31fff62c51ff04ed06ff72f28 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index bcd0a39e..fa881ae5 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit bcd0a39e94fcfd64e77e0700366dda7b1e04ec22 +Subproject commit fa881ae5dac60ab31fff62c51ff04ed06ff72f28 From 2f632fbce08d2b2839a8d034f6bc8e88de773b90 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 9 Jan 2026 15:24:36 -0700 Subject: [PATCH 147/150] Update atmospheric_physics hash to use cleaned-up code and updated CAM4 SDF. --- .gitmodules | 2 +- src/physics/ncar_ccpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 309b91cd..fcb9a80b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/nusbaume/atmospheric_physics - fxtag = fa881ae5dac60ab31fff62c51ff04ed06ff72f28 + fxtag = 6666bd5ea45f498226ad12a2f83f4ec9a5e5b4f4 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index fa881ae5..6666bd5e 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit fa881ae5dac60ab31fff62c51ff04ed06ff72f28 +Subproject commit 6666bd5ea45f498226ad12a2f83f4ec9a5e5b4f4 From 53b40bba5f89c09111867d4dd1dd85ffeeba4b75 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 9 Jan 2026 16:21:32 -0700 Subject: [PATCH 148/150] Fix issues found while opening the PR. --- src/control/cam_control_mod.F90 | 3 --- src/dynamics/mpas/dyn_coupling_impl.F90 | 1 - src/utils/machine.F90 | 12 ------------ 3 files changed, 16 deletions(-) delete mode 100644 src/utils/machine.F90 diff --git a/src/control/cam_control_mod.F90 b/src/control/cam_control_mod.F90 index 34ca3c3e..d135eea8 100644 --- a/src/control/cam_control_mod.F90 +++ b/src/control/cam_control_mod.F90 @@ -101,7 +101,6 @@ end subroutine cam_ctrl_init !--------------------------------------------------------------------------- subroutine cam_ctrl_set_orbit(eccen_in, obliqr_in, lambm0_in, mvelpp_in) - use phys_vars_init_check, only: mark_as_initialized real(r8), intent(in) :: eccen_in real(r8), intent(in) :: obliqr_in @@ -113,8 +112,6 @@ subroutine cam_ctrl_set_orbit(eccen_in, obliqr_in, lambm0_in, mvelpp_in) lambm0 = lambm0_in mvelpp = mvelpp_in - call mark_as_initialized('planet_orbital_eccentricity_factor') - end subroutine cam_ctrl_set_orbit end module cam_control_mod diff --git a/src/dynamics/mpas/dyn_coupling_impl.F90 b/src/dynamics/mpas/dyn_coupling_impl.F90 index 60438c45..99da7314 100644 --- a/src/dynamics/mpas/dyn_coupling_impl.F90 +++ b/src/dynamics/mpas/dyn_coupling_impl.F90 @@ -117,7 +117,6 @@ module subroutine dyn_exchange_constituent_states(direction, exchange, conversio 'sigma_all_q(pver)', & file='dyn_coupling', line=__LINE__, errmsg=trim(adjustl(cerr))) - !constituents => cam_constituents_array() constituents => cam_advected_constituents_array() if (.not. associated(constituents)) then diff --git a/src/utils/machine.F90 b/src/utils/machine.F90 deleted file mode 100644 index 8d186ad8..00000000 --- a/src/utils/machine.F90 +++ /dev/null @@ -1,12 +0,0 @@ -! This module is the CAM-SIMA version of the CCPP generated module of the same name -module machine - - use ccpp_kinds, only: kind_phys => kind_phys - - - implicit none - private - - public kind_phys - -end module machine From bd2160fdd5889e4eefc9e7ee952c0e402645e1d7 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Wed, 21 Jan 2026 14:49:28 -0700 Subject: [PATCH 149/150] Update submodule and XML files based on initial reviewer comments. --- .gitmodules | 2 +- cime_config/namelist_definition_cam.xml | 72 ------------------------- cime_config/testdefs/testlist_cam.xml | 9 ---- src/physics/ncar_ccpp | 2 +- 4 files changed, 2 insertions(+), 83 deletions(-) diff --git a/.gitmodules b/.gitmodules index fcb9a80b..7158737c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,7 +20,7 @@ [submodule "ncar-physics"] path = src/physics/ncar_ccpp url = https://github.com/nusbaume/atmospheric_physics - fxtag = 6666bd5ea45f498226ad12a2f83f4ec9a5e5b4f4 + fxtag = 45ed0c1ca48ef19f0558f72191b362f1d19ac921 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/cime_config/namelist_definition_cam.xml b/cime_config/namelist_definition_cam.xml index 0237d591..f6b8900d 100644 --- a/cime_config/namelist_definition_cam.xml +++ b/cime_config/namelist_definition_cam.xml @@ -20,78 +20,6 @@ ${DIN_LOC_ROOT}/atm/cam/inic/cam_vcoords_L30_c180105.nc ${DIN_LOC_ROOT}/atm/cam/inic/cam_vcoords_L32_c180105.nc - - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-01-01_0.23x0.31_L26_c100513.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-09-01_0.23x0.31_L26_c061106.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_1980-01-01_0.47x0.63_L26_c071226.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-09-01_0.47x0.63_L26_c061106.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-10-01_0.5x0.625_L26_c031204.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_1987-01-01_0.9x1.25_L26_c060703.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-09-01_0.9x1.25_L26_c051205.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-01-01_1.9x2.5_L26_c070408.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-09-01_1.9x2.5_L26_c040809.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-01-01_2.5x3.33_L26_c110309.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-09-01_2.5x3.33_L26_c091007.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0001-01-01_4x5_L26_c060608.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-01-01_10x15_L26_c030918.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami-mam3_0000-01-01_0.23x0.31_L30_c110527.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami-mam3_0000-01-01_0.47x0.63_L30_c100929.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami-mam3_0000-01-01_0.9x1.25_L30_c100618.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami-mam3_0000-01-01_1.9x2.5_L30_c090306.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-09-01_1.9x2.5_L30_c070109.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-01-01_2.5x3.33_L30_c110309.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-09-01_2.5x3.33_L30_c100831.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-01-01_4x5_L30_c090108.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-01-01_10x15_L30_c081013.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami-mam3_0000-01-01_0.9x1.25_L32_c141031.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami-mam3_0000-01-01_1.9x2.5_L32_c150407.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami-mam4_0000-01-01_10x15_L32_c170914.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami_0000-01-01_0.47x0.63_L26_APE_c080227.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/aqua_0006-01-01_0.9x1.25_L26_c161020.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/aqua_0006-01-01_1.9x2.5_L26_c161020.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/aqua_0000-01-01_10x15_L26_c161230.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/aqua_0006-01-01_0.9x1.25_L30_c161020.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/aqua_0006-01-01_1.9x2.5_L30_c161020.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/aqua_0000-01-01_10x15_L30_c170103.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/aqua_0006-01-01_0.9x1.25_L32_c161020.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/aqua_0006-01-01_1.9x2.5_L32_c161020.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/aqua_0000-01-01_10x15_L32_c170103.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami-chem_1990-01-01_0.9x1.25_L30_c080724.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami-chem_1990-01-01_1.9x2.5_L26_c080114.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/cami-chem_1990-01-01_1.9x2.5_L30_c080215.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/camchemi_0012-01-01_10x15_L26_c081104.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/camchemi_0012-01-01_10x15_L30_c081104.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/camchemi_0012-01-01_4x5_L26_c081104.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/camchemi_0012-01-01_4x5_L30_c081104.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/camchemi_0012-01-01_1.9x2.5_L26_c081104.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/camchemi_0012-01-01_1.9x2.5_L30_c081104.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/trop_strat_mam3_chem_2000-01-01_10x15_L30_c121015.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/trop_strat_mam3_chem_2000-01-01_4x5_L30_c121015.nc - ${DIN_LOC_ROOT}/atm/cam/inic/fv/trop_strat_mam3_chem_2000-01-01_1.9x2.5_L30_c121015.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart/ic/cami_0000-09-01_4x5_L26_c060217.nc - ${DIN_LOC_ROOT}/atm/cam/chem/trop_mozart/ic/cami_0000-09-01_10x15_L26_c060216.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/cami_2000-02-01_0.9x1.25_L66_c040928.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/cami_2000-07-01_1.9x2.5_L66_c040928.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/FWT2000_f09_spinup01.cam.i.0001-01-02-00000_c160315.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/f2000.waccm-mam3_1.9x2.5_L70.cam2.i.0017-01-01.c120410.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/f2000.waccm-mam3_10x15_L70.cam2.i.0017-01-01.c141016.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/b1850.waccm-mam3_1.9x2.5_L70.cam2.i.0156-01-01.c120523.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/cami_2000-05-01_1.9x2.5_L103_c040928.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/wa3_4x5_1950_spinup.cam2.i.1960-01-01-00000.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/cami_2000-01-01_10x15_L66_c041121.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/f40.2000.4deg.wcm.carma.sulf.004.cam2.i.0008-01-01-00000.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/f40.2deg.wcm.carma.sulf.L66.cam2.i.2010-01-01.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/WAX3548T08CO_2003top_f2000.waccm_0017bottom_L81_c110906.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/WAX3548T08CO_2003top_f2000.waccm_0017bottom_4x5_L81_c160630.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/WAX3548T08CO_2003top_f2000.waccm_0017bottom_10x15_L81_c141027.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/waccmx_aqua_4x5_L126_c170705.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/fx2000_0.9x1.25_126lev_0002-01-01-00000_c181221.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/wcmx-cam6-phys_1.9x2.5_130lev_2000_c181115.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/wcmx-cam6-phys_0.9x1.25_130lev_2000_c190122.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/FC6X2000_f05_spinup01.cam.i.0002-01-01-00000_c190711.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/waccmx_mam4_aqua_4x5_L130_c180803.nc - ${DIN_LOC_ROOT}/atm/waccm/ic/waccmx_mam4_aqua_1.9x2.5_L130_c180803.nc - ${DIN_LOC_ROOT}/atm/cam/inic/se/FCts4MTHIST_ne3pg3_spinup02.cam.i.1980-01-01_c240702.nc ${DIN_LOC_ROOT}/atm/cam/inic/se/cam6_QPC6_topo_ne3pg3_mg37_L32_01-01-31_c221214.nc diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index c804c8ea..81542fff 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -88,15 +88,6 @@ - - - - - - - - - diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 6666bd5e..45ed0c1c 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 6666bd5ea45f498226ad12a2f83f4ec9a5e5b4f4 +Subproject commit 45ed0c1ca48ef19f0558f72191b362f1d19ac921 From 8258bf52d61d0cd9027a2d3d0fba67a0070460ef Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Thu, 22 Jan 2026 10:09:17 -0700 Subject: [PATCH 150/150] Update atmospheric_physics branch to use official repo. --- .gitmodules | 4 ++-- src/physics/ncar_ccpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 7158737c..925620f8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,8 +19,8 @@ fxDONOTUSEurl = https://github.com/MPAS-Dev/MPAS-Model.git [submodule "ncar-physics"] path = src/physics/ncar_ccpp - url = https://github.com/nusbaume/atmospheric_physics - fxtag = 45ed0c1ca48ef19f0558f72191b362f1d19ac921 + url = https://github.com/ESCOMP/atmospheric_physics + fxtag = b47246dedfa7c377ac2bfb060e2d465beb931f2a fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp index 45ed0c1c..b47246de 160000 --- a/src/physics/ncar_ccpp +++ b/src/physics/ncar_ccpp @@ -1 +1 @@ -Subproject commit 45ed0c1ca48ef19f0558f72191b362f1d19ac921 +Subproject commit b47246dedfa7c377ac2bfb060e2d465beb931f2a