From b3b248deacf27397667eb9232a30849be4eac4c1 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Mon, 2 Mar 2026 16:44:23 -0700 Subject: [PATCH 01/15] enable average fields --- .gitmodules | 4 ++-- cime_config/atm_in_paramgen.py | 2 +- src/history/buffers | 2 +- src/history/cam_hist_file.F90 | 9 +++++---- src/history/cam_history.F90 | 3 ++- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.gitmodules b/.gitmodules index 6bb16a2c3..2eae362f5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,8 +6,8 @@ fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework [submodule "history"] path = src/history/buffers - url = https://github.com/ESMCI/history_output - fxtag = history01_00 + url = https://github.com/peverwhee/history_output + fxtag = 1404b7d0b602a36c6bc5abdf4b1ef3d2af4b1a50 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESMCI/history_output [submodule "mpas"] diff --git a/cime_config/atm_in_paramgen.py b/cime_config/atm_in_paramgen.py index 5fb1aea1d..6682ac622 100644 --- a/cime_config/atm_in_paramgen.py +++ b/cime_config/atm_in_paramgen.py @@ -1410,7 +1410,7 @@ def append_user_nl_file(self, user_nl_file): #End if (array indices) #Extract value string: - val_str = ' '.join(line_ss[1:]) # the rest is the value string + val_str = ' '.join(line_ss[1:]).strip() # the rest is the value string #Check if value string ends in array continuation: if is_array: diff --git a/src/history/buffers b/src/history/buffers index f59357ea6..1404b7d0b 160000 --- a/src/history/buffers +++ b/src/history/buffers @@ -1 +1 @@ -Subproject commit f59357ea6d96cfe949ba5b08995a48dff4b1b987 +Subproject commit 1404b7d0b602a36c6bc5abdf4b1ef3d2af4b1a50 diff --git a/src/history/cam_hist_file.F90 b/src/history/cam_hist_file.F90 index b499dd9b7..b86f2fc90 100644 --- a/src/history/cam_hist_file.F90 +++ b/src/history/cam_hist_file.F90 @@ -762,7 +762,6 @@ subroutine config_set_up_fields(this, possible_field_list) write(errmsg,'(3a)') 'ERROR Field : ',trim(this%field_names(idx)),' not available' call endrun(subname//errmsg, file=__FILE__, line=__LINE__) end select - !call field_ptr%dimensions(dimensions) dimensions = field_ptr%dimensions() field_shape = field_ptr%shape() beg_dim = field_ptr%beg_dims() @@ -1505,6 +1504,7 @@ subroutine config_write_field(this, field, split_file_index, restart, & use cam_abortutils, only: check_allocate, endrun use hist_field, only: hist_field_info_t use shr_kind_mod, only: r4 => shr_kind_r4 + use hist_msg_handler, only: hist_log_messages ! Dummy arguments class(hist_file_t), intent(inout) :: this type(hist_field_info_t), intent(inout) :: field @@ -1529,6 +1529,7 @@ subroutine config_write_field(this, field, split_file_index, restart, & real(r8), allocatable :: field_data_r8(:,:) real(r4), allocatable :: field_data_r4(:,:) class(hist_buffer_t), pointer :: buff_ptr + type(hist_log_messages) :: errors character(len=CL) :: errmsg character(len=*), parameter :: subname = 'config_write_field: ' @@ -1594,6 +1595,9 @@ subroutine config_write_field(this, field, split_file_index, restart, & end if end do + ! Clear the buffers after writing + call field%clear_buffers(logger=errors) + end subroutine config_write_field ! ======================================================================== @@ -1643,7 +1647,6 @@ subroutine config_clear_buffers(this) integer :: field_idx type(hist_log_messages) :: errors - do field_idx = 1, size(this%field_list) call this%field_list(field_idx)%clear_buffers(logger=errors) if (masterproc) then @@ -1802,8 +1805,6 @@ subroutine read_namelist_entry(unitn, hfile_config, hist_inst_fields, & masterprocid, mpicom, ierr) end if if (num_fields_avg > 0) then - call endrun(subname//"ERROR, average fields not yet implemented", & - file=__FILE__, line=__LINE__) call MPI_Bcast(hist_avg_fields(:), max_fldlen*num_fields_avg, MPI_CHARACTER, & masterprocid, mpicom, ierr) end if diff --git a/src/history/cam_history.F90 b/src/history/cam_history.F90 index 7823ed232..cf1f07bd1 100644 --- a/src/history/cam_history.F90 +++ b/src/history/cam_history.F90 @@ -179,6 +179,8 @@ subroutine history_write_files() write_history = .true. else write_history = .false. + ! Zero out the history buffers so nothing gets accumulated from nstep0 + call hist_configs(file_idx)%clear_buffers() end if end if if (.not. write_history) then @@ -838,7 +840,6 @@ subroutine history_wrap_up(restart_write, last_timestep) write(iulog,9004) end if end if - call hist_configs(file_idx)%clear_buffers() end do 9003 format(' Output at NSTEP = ',i10,/, & ' Number of time samples on this file = ',i10,/, & From a224991e916434320cb71684c9208579abcba705 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Mon, 9 Mar 2026 11:11:34 -0600 Subject: [PATCH 02/15] write nstep0 separately --- src/history/cam_hist_file.F90 | 11 +++++++++++ src/history/cam_history.F90 | 6 +++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/history/cam_hist_file.F90 b/src/history/cam_hist_file.F90 index b86f2fc90..bfc7aee77 100644 --- a/src/history/cam_hist_file.F90 +++ b/src/history/cam_hist_file.F90 @@ -126,6 +126,7 @@ module cam_hist_file procedure :: write_field => config_write_field procedure :: close_files => config_close_files procedure :: clear_buffers => config_clear_buffers + procedure :: reset_samples => config_reset_samples end type hist_file_t private :: count_array ! Number of non-blank strings in array @@ -1658,6 +1659,16 @@ end subroutine config_clear_buffers ! ======================================================================== + subroutine config_reset_samples(this) + + class(hist_file_t), intent(inout) :: this + + this%num_samples = 0 + + end subroutine config_reset_samples + +! ======================================================================== + pure function count_array(arr_in) result(arr_count) ! Dummy arguments character(len=*), intent(in) :: arr_in(:) diff --git a/src/history/cam_history.F90 b/src/history/cam_history.F90 index cf1f07bd1..a963f244b 100644 --- a/src/history/cam_history.F90 +++ b/src/history/cam_history.F90 @@ -213,6 +213,10 @@ subroutine history_write_files() call hist_configs(file_idx)%define_file(restart, logname, host, model_doi_url) end if call hist_configs(file_idx)%write_time_dependent_variables(restart) + if (nstep == 0) then + ! Reset samples if nstep0 was written + call hist_configs(file_idx)%reset_samples() + end if end do end subroutine history_write_files @@ -811,7 +815,7 @@ subroutine history_wrap_up(restart_write, last_timestep) full = .false. num_samples = hist_configs(file_idx)%get_num_samples() max_frames = hist_configs(file_idx)%max_frame() - if (mod(num_samples, max_frames) == 0 .and. num_samples > 0) then + if (nstep == 0 .or. (mod(num_samples, max_frames) == 0 .and. num_samples > 0)) then full = .true. end if if ((full .or. (last_timestep .and. num_samples >= 1)) .and. & From ee32aa0582c97fd39d5aa8b37f78cddcf6a51fb0 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Mon, 23 Mar 2026 16:36:10 -0600 Subject: [PATCH 03/15] add min and max --- src/history/cam_hist_file.F90 | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/history/cam_hist_file.F90 b/src/history/cam_hist_file.F90 index bfc7aee77..0fccb5de4 100644 --- a/src/history/cam_hist_file.F90 +++ b/src/history/cam_hist_file.F90 @@ -1820,14 +1820,10 @@ subroutine read_namelist_entry(unitn, hfile_config, hist_inst_fields, & masterprocid, mpicom, ierr) end if if (num_fields_min > 0) then - call endrun(subname//"ERROR, minimum fields not yet implemented", & - file=__FILE__, line=__LINE__) call MPI_Bcast(hist_min_fields(:), max_fldlen*num_fields_min, MPI_CHARACTER, & masterprocid, mpicom, ierr) end if if (num_fields_max > 0) then - call endrun(subname//"ERROR, maximum fields not yet implemented", & - file=__FILE__, line=__LINE__) call MPI_Bcast(hist_max_fields(:), max_fldlen*num_fields_max, MPI_CHARACTER, & masterprocid, mpicom, ierr) end if From 17888123bc870f2adadb3b53fd8b3d4ca9d4c5cb Mon Sep 17 00:00:00 2001 From: peverwhee Date: Thu, 2 Apr 2026 16:00:40 -0600 Subject: [PATCH 04/15] add fill value logic --- src/history/cam_hist_file.F90 | 47 ++++++++++++++++++++++++++--------- src/history/cam_history.F90 | 37 ++++++++++++++++++++++----- 2 files changed, 66 insertions(+), 18 deletions(-) diff --git a/src/history/cam_hist_file.F90 b/src/history/cam_hist_file.F90 index 83a0c87f2..d209bdb62 100644 --- a/src/history/cam_hist_file.F90 +++ b/src/history/cam_hist_file.F90 @@ -770,9 +770,10 @@ subroutine config_set_up_fields(this, possible_field_list) field_info => hist_new_field(this%field_names(idx), & field_ptr%standard_name(), field_ptr%long_name(), & field_ptr%units(), field_ptr%type(), field_ptr%decomp(), & - dimensions, this%accumulate_types(idx), field_ptr%num_levels(), & - field_shape, beg_dims=beg_dim, end_dims=end_dim, & - mixing_ratio=field_ptr%mixing_ratio(), & + dimensions, this%accumulate_types(idx), & + field_ptr%num_levels(), field_shape, & + field_ptr%fill_value(), beg_dims=beg_dim, & + end_dims=end_dim, mixing_ratio=field_ptr%mixing_ratio(), & flag_xyfill=field_ptr%flag_xyfill(), & sampling_seq=field_ptr%sampling_sequence()) call hist_new_buffer(field_info, field_shape, & @@ -914,7 +915,6 @@ subroutine config_define_file(this, restart, logname, host, model_doi_url) use cam_history_support, only: write_hist_coord_attrs use cam_history_support, only: write_hist_coord_vars use cam_history_support, only: max_chars - use cam_history_support, only: fillvalue use shr_kind_mod, only: r4 => shr_kind_r4 use time_manager, only: get_ref_date, timemgr_get_calendar_cf use time_manager, only: get_step_size @@ -1293,14 +1293,14 @@ subroutine config_define_file(this, restart, logname, host, model_doi_url) ! netCDF-aware tools (ncview, ncdump, etc.) recognise fill ! values. The attribute type must match the variable type. if (ncreal == pio_double) then - ierr = pio_put_att(this%hist_files(split_file_index), varid, '_FillValue', fillvalue) + ierr = pio_put_att(this%hist_files(split_file_index), varid, '_FillValue', this%field_list(field_index)%fill_value()) call cam_pio_handle_error(ierr, subname//'cannot define _FillValue for '//trim(fname_tmp)) - ierr = pio_put_att(this%hist_files(split_file_index), varid, 'missing_value', fillvalue) + ierr = pio_put_att(this%hist_files(split_file_index), varid, 'missing_value', this%field_list(field_index)%fill_value()) call cam_pio_handle_error(ierr, subname//'cannot define missing_value for '//trim(fname_tmp)) else - ierr = pio_put_att(this%hist_files(split_file_index), varid, '_FillValue', real(fillvalue, r4)) + ierr = pio_put_att(this%hist_files(split_file_index), varid, '_FillValue', real(this%field_list(field_index)%fill_value(), r4)) call cam_pio_handle_error(ierr, subname//'cannot define _FillValue for '//trim(fname_tmp)) - ierr = pio_put_att(this%hist_files(split_file_index), varid, 'missing_value', real(fillvalue, r4)) + ierr = pio_put_att(this%hist_files(split_file_index), varid, 'missing_value', real(this%field_list(field_index)%fill_value(), r4)) call cam_pio_handle_error(ierr, subname//'cannot define missing_value for '//trim(fname_tmp)) end if end if @@ -1522,6 +1522,8 @@ subroutine config_write_field(this, field, split_file_index, restart, & use hist_field, only: hist_field_info_t use shr_kind_mod, only: r4 => shr_kind_r4 use hist_msg_handler, only: hist_log_messages + use cam_logfile, only: iulog + use cam_abortutils, only: endrun ! Dummy arguments class(hist_file_t), intent(inout) :: this type(hist_field_info_t), intent(inout) :: field @@ -1591,21 +1593,42 @@ subroutine config_write_field(this, field, split_file_index, restart, & buff_ptr => field%buffers if (frank == 1) then if (trim(field_precision) == 'REAL32') then - call hist_buffer_norm_value(buff_ptr, field_data_r4(:,1)) + call hist_buffer_norm_value(buff_ptr, field_data_r4(:,1), logger=errors) + if (errors%num_errors() > 0) then + call errors%output(iulog) + write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' + call endrun(errmsg) + end if call cam_grid_write_dist_array(this%hist_files(split_file_index), field_decomp, (/dim_sizes(1)/), & field_shape, field_data_r4(:,1), varid) else - call hist_buffer_norm_value(buff_ptr, field_data_r8(:,1)) + call hist_buffer_norm_value(buff_ptr, field_data_r8(:,1), logger=errors) + if (errors%num_errors() > 0) then + call errors%output(iulog) + write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' + call endrun(errmsg) + end if call cam_grid_write_dist_array(this%hist_files(split_file_index), field_decomp, (/dim_sizes(1)/), & field_shape, field_data_r8(:,1), varid) end if else if (trim(field_precision) == 'REAL32') then - call hist_buffer_norm_value(buff_ptr, field_data_r4) + call hist_buffer_norm_value(buff_ptr, field_data_r4, logger=errors) + if (errors%num_errors() > 0) then + call errors%output(iulog) + write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' + call endrun(errmsg) + end if call cam_grid_write_dist_array(this%hist_files(split_file_index), field_decomp, dim_sizes(1:frank), & field_shape, field_data_r4, varid) else - call hist_buffer_norm_value(buff_ptr, field_data_r8) + call hist_buffer_norm_value(buff_ptr, field_data_r8, logger=errors) + call errors%output(iulog) + if (errors%num_errors() > 0) then + call errors%output(iulog) + write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' + call endrun(errmsg) + end if call cam_grid_write_dist_array(this%hist_files(split_file_index), field_decomp, dim_sizes(1:frank), & field_shape, field_data_r8, varid) end if diff --git a/src/history/cam_history.F90 b/src/history/cam_history.F90 index 8a7ebabad..a023b5574 100644 --- a/src/history/cam_history.F90 +++ b/src/history/cam_history.F90 @@ -382,9 +382,10 @@ end subroutine set_up_field_list_hash_table !=========================================================================== subroutine history_add_field_1d(diagnostic_name, standard_name, vdim_name, & - avgflag, units, gridname, flag_xyfill, mixing_ratio) + avgflag, units, gridname, flag_xyfill, fill_value, mixing_ratio) use cam_history_support, only: get_hist_coord_index, max_chars, horiz_only use cam_abortutils, only: endrun, check_allocate + use shr_kind_mod, only: r8 => shr_kind_r8 !----------------------------------------------------------------------- ! ! Purpose: Add a field to the master field list @@ -405,6 +406,7 @@ subroutine history_add_field_1d(diagnostic_name, standard_name, vdim_name, & character(len=*), intent(in) :: units ! units of fname (max_chars) character(len=*), optional, intent(in) :: gridname logical, optional, intent(in) :: flag_xyfill + real(r8), optional, intent(in) :: fill_value character(len=*), optional, intent(in) :: mixing_ratio ! @@ -430,14 +432,14 @@ subroutine history_add_field_1d(diagnostic_name, standard_name, vdim_name, & dimnames(1) = trim(vdim_name) end if call history_add_field(diagnostic_name, standard_name, dimnames, avgflag, units, & - gridname=gridname, flag_xyfill=flag_xyfill, mixing_ratio=mixing_ratio) + gridname=gridname, flag_xyfill=flag_xyfill, fill_value=fill_value, mixing_ratio=mixing_ratio) end subroutine history_add_field_1d !=========================================================================== subroutine history_add_field_nd(diagnostic_name, standard_name, dimnames, avgflag, & - units, gridname, flag_xyfill, mixing_ratio) + units, gridname, flag_xyfill, fill_value, mixing_ratio) !----------------------------------------------------------------------- ! ! Purpose: Add a field to the master field list - generic; called from @@ -453,7 +455,7 @@ subroutine history_add_field_nd(diagnostic_name, standard_name, dimnames, avgfla use cam_grid_support, only: cam_grid_dimensions use cam_grid_support, only: cam_grid_id, cam_grid_is_zonal use cam_grid_support, only: cam_grid_get_array_bounds - use cam_history_support, only: lookup_hist_coord_indices + use cam_history_support, only: lookup_hist_coord_indices, fillvalue use cam_history_support, only: hist_coord_find_levels, hist_coords use cam_history_support, only: max_fldlen=>max_fieldname_len, max_chars, fieldname_len use cam_hist_file, only: strip_suffix @@ -461,6 +463,7 @@ subroutine history_add_field_nd(diagnostic_name, standard_name, dimnames, avgfla use cam_abortutils, only: endrun, check_allocate use spmd_utils, only: masterproc use string_utils, only: stringify + use shr_kind_mod, only: r8 => shr_kind_r8 character(len=*), intent(in) :: diagnostic_name character(len=*), intent(in) :: standard_name @@ -469,6 +472,7 @@ subroutine history_add_field_nd(diagnostic_name, standard_name, dimnames, avgfla character(len=*), intent(in) :: units ! units of fname (max_chars) character(len=*), optional, intent(in) :: gridname logical, optional, intent(in) :: flag_xyfill + real(r8), optional, intent(in) :: fill_value character(len=*), optional, intent(in) :: mixing_ratio ! Local variables @@ -488,6 +492,7 @@ subroutine history_add_field_nd(diagnostic_name, standard_name, dimnames, avgfla character(len=max_fldlen) :: cell_methods character(len=3) :: mixing_ratio_loc character(len=*), parameter :: subname = 'history_add_field_nd: ' + real(r8) :: fill_value_local if (size(hist_configs) > 0) then if (hist_configs(1)%file_is_setup()) then @@ -545,6 +550,12 @@ subroutine history_add_field_nd(diagnostic_name, standard_name, dimnames, avgfla call endrun('history_add_field_nd: '//trim(errmsg)) end if + if (present(fill_value)) then + fill_value_local = fill_value + else + fill_value_local = fillvalue + end if + ! Indicate if some field pre-processing occurred (e.g., zonal mean) if (cam_grid_is_zonal(grid_decomp)) then call cam_grid_get_coord_names(grid_decomp, coord_name, errmsg) @@ -599,7 +610,8 @@ subroutine history_add_field_nd(diagnostic_name, standard_name, dimnames, avgfla else field_ptr%next => hist_new_field(diagnostic_name, & standard_name, standard_name, units, 'real', grid_decomp, & - mdim_indices, avgflag, num_levels, field_shape, mixing_ratio=mixing_ratio_loc, & + mdim_indices, avgflag, num_levels, field_shape, fill_value_local, & + mixing_ratio=mixing_ratio_loc, & dim_bounds=dimbounds, mdim_sizes=mdim_sizes, cell_methods=cell_methods, & flag_xyfill=flag_xyfill) exit @@ -608,7 +620,8 @@ subroutine history_add_field_nd(diagnostic_name, standard_name, dimnames, avgfla else possible_field_list_head => hist_new_field(diagnostic_name, & standard_name, standard_name, units, 'real', grid_decomp, & - mdim_indices, avgflag, num_levels, field_shape, mixing_ratio=mixing_ratio_loc, & + mdim_indices, avgflag, num_levels, field_shape, fill_value_local, & + mixing_ratio=mixing_ratio_loc, & dim_bounds=dimbounds, mdim_sizes=mdim_sizes, cell_methods=cell_methods, & flag_xyfill=flag_xyfill) end if @@ -658,6 +671,10 @@ subroutine history_out_field_1d(diagnostic_name, field_values) call hist_field_accumulate(field_info, field_values, 1, logger=logger) if (masterproc) then call logger%output(iulog) + if (logger%num_errors() > 0) then + write(errmsg, *) subname, 'Error during hist_field_accumulate for "', trim(diagnostic_name), '"' + call endrun(errmsg) + end if end if end do @@ -705,6 +722,10 @@ subroutine history_out_field_2d(diagnostic_name, field_values) call hist_field_accumulate(field_info, field_values, 1, logger=logger) if (masterproc) then call logger%output(iulog) + if (logger%num_errors() > 0) then + write(errmsg, *) subname, 'Error during hist_field_accumulate for "', trim(diagnostic_name), '"' + call endrun(errmsg) + end if end if end do @@ -749,6 +770,10 @@ subroutine history_out_field_3d(diagnostic_name, field_values) !call hist_field_accumulate(field_info, real(field_values, REAL64), 1) !if (masterproc) then ! call logger%output(iulog) + ! if (logger%num_errors() > 0) then + ! write(errmsg, *) subname, 'Error during hist_field_accumulate for "', trim(diagnostic_name), '"' + ! call endrun(errmsg) + ! end if !end if end do From bb2308a24728cd57c908fb2310916bc7844660c3 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Thu, 2 Apr 2026 16:02:50 -0600 Subject: [PATCH 05/15] update buffers tag --- .gitmodules | 2 +- src/history/buffers | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index f90667c2d..d8def8817 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,7 +7,7 @@ [submodule "history"] path = src/history/buffers url = https://github.com/peverwhee/history_output - fxtag = 1404b7d0b602a36c6bc5abdf4b1ef3d2af4b1a50 + fxtag = 1c428a68bfeb448ba1399bd94d3ca3280838bcbf fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESMCI/history_output [submodule "mpas"] diff --git a/src/history/buffers b/src/history/buffers index 1404b7d0b..1c428a68b 160000 --- a/src/history/buffers +++ b/src/history/buffers @@ -1 +1 @@ -Subproject commit 1404b7d0b602a36c6bc5abdf4b1ef3d2af4b1a50 +Subproject commit 1c428a68bfeb448ba1399bd94d3ca3280838bcbf From 15d45e29841a860b2f705acbc6d87b513c9c219c Mon Sep 17 00:00:00 2001 From: peverwhee Date: Mon, 6 Apr 2026 15:44:25 -0600 Subject: [PATCH 06/15] add stdev --- .gitmodules | 2 +- src/history/buffers | 2 +- src/history/cam_hist_file.F90 | 5 ----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index d8def8817..dd6fd0137 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,7 +7,7 @@ [submodule "history"] path = src/history/buffers url = https://github.com/peverwhee/history_output - fxtag = 1c428a68bfeb448ba1399bd94d3ca3280838bcbf + fxtag = 793b8f08d5b1090dd9ac6ea25820c5406b605230 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESMCI/history_output [submodule "mpas"] diff --git a/src/history/buffers b/src/history/buffers index 1c428a68b..793b8f08d 160000 --- a/src/history/buffers +++ b/src/history/buffers @@ -1 +1 @@ -Subproject commit 1c428a68bfeb448ba1399bd94d3ca3280838bcbf +Subproject commit 793b8f08d5b1090dd9ac6ea25820c5406b605230 diff --git a/src/history/cam_hist_file.F90 b/src/history/cam_hist_file.F90 index d209bdb62..892fe6363 100644 --- a/src/history/cam_hist_file.F90 +++ b/src/history/cam_hist_file.F90 @@ -781,9 +781,6 @@ subroutine config_set_up_fields(this, possible_field_list) if (masterproc) then call errors%output(iulog) end if - ! peverwhee - TODO: create additional buffer(s) for other accum types -! call hist_new_buffer(field_info, field_shape, & -! this%rl_kind, 1, this%accumulate_types(idx), 1) ! Add to field list array and hash table this%field_list(idx) = field_info call this%field_list_hash_table%add_hash_key(field_info) @@ -1867,8 +1864,6 @@ subroutine read_namelist_entry(unitn, hfile_config, hist_inst_fields, & masterprocid, mpicom, ierr) end if if (num_fields_var > 0) then - call endrun(subname//"ERROR, standard deviation fields not yet implemented", & - file=__FILE__, line=__LINE__) call MPI_Bcast(hist_var_fields(:), max_fldlen*num_fields_var, MPI_CHARACTER, & masterprocid, mpicom, ierr) end if From d771fe779820b7c6bbe5f6b2349525cc0494ff03 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Tue, 7 Apr 2026 14:55:05 -0600 Subject: [PATCH 07/15] do accumulation with r8 data --- .gitmodules | 2 +- src/history/buffers | 2 +- src/history/cam_hist_file.F90 | 46 +++++++++++++++-------------------- 3 files changed, 22 insertions(+), 28 deletions(-) diff --git a/.gitmodules b/.gitmodules index dd6fd0137..0f8b90d4e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,7 +7,7 @@ [submodule "history"] path = src/history/buffers url = https://github.com/peverwhee/history_output - fxtag = 793b8f08d5b1090dd9ac6ea25820c5406b605230 + fxtag = b213964573e0e87b7a10acf627f1b435643a0b78 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESMCI/history_output [submodule "mpas"] diff --git a/src/history/buffers b/src/history/buffers index 793b8f08d..b21396457 160000 --- a/src/history/buffers +++ b/src/history/buffers @@ -1 +1 @@ -Subproject commit 793b8f08d5b1090dd9ac6ea25820c5406b605230 +Subproject commit b213964573e0e87b7a10acf627f1b435643a0b78 diff --git a/src/history/cam_hist_file.F90 b/src/history/cam_hist_file.F90 index 892fe6363..938b138ae 100644 --- a/src/history/cam_hist_file.F90 +++ b/src/history/cam_hist_file.F90 @@ -778,8 +778,9 @@ subroutine config_set_up_fields(this, possible_field_list) sampling_seq=field_ptr%sampling_sequence()) call hist_new_buffer(field_info, field_shape, & this%rl_kind, 1, this%accumulate_types(idx), 1, errors=errors) - if (masterproc) then + if (masterproc .and. errors%num_errors() > 0) then call errors%output(iulog) + call endrun(subname//' error(s) during buffer creation') end if ! Add to field list array and hash table this%field_list(idx) = field_info @@ -1559,6 +1560,8 @@ subroutine config_write_field(this, field, split_file_index, restart, & if (trim(field_precision) == 'REAL32') then allocate(field_data_r4(end_dims(1) - beg_dims(1) + 1, 1), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'field_data_r4', file=__FILE__, line=__LINE__-1, errmsg=errmsg) + allocate(field_data_r8(end_dims(1) - beg_dims(1) + 1, 1), stat=ierr, errmsg=errmsg) + call check_allocate(ierr, subname, 'field_data_r8', file=__FILE__, line=__LINE__-1, errmsg=errmsg) else allocate(field_data_r8(end_dims(1) - beg_dims(1) + 1, 1), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'field_data_r8', file=__FILE__, line=__LINE__-1, errmsg=errmsg) @@ -1567,6 +1570,8 @@ subroutine config_write_field(this, field, split_file_index, restart, & if (trim(field_precision) == 'REAL32') then allocate(field_data_r4(end_dims(1) - beg_dims(1) + 1, field_shape(2)), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'field_data_r4', file=__FILE__, line=__LINE__-1, errmsg=errmsg) + allocate(field_data_r8(end_dims(1) - beg_dims(1) + 1, field_shape(2)), stat=ierr, errmsg=errmsg) + call check_allocate(ierr, subname, 'field_data_r8', file=__FILE__, line=__LINE__-1, errmsg=errmsg) else allocate(field_data_r8(end_dims(1) - beg_dims(1) + 1, field_shape(2)), stat=ierr, errmsg=errmsg) call check_allocate(ierr, subname, 'field_data_r8', file=__FILE__, line=__LINE__-1, errmsg=errmsg) @@ -1589,43 +1594,32 @@ subroutine config_write_field(this, field, split_file_index, restart, & call pio_setframe(this%hist_files(split_file_index), varid, int(sample_index,kind=PIO_OFFSET_KIND)) buff_ptr => field%buffers if (frank == 1) then + call hist_buffer_norm_value(buff_ptr, field_data_r8(:,1), logger=errors) + if (errors%num_errors() > 0) then + call errors%output(iulog) + write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' + call endrun(errmsg) + end if if (trim(field_precision) == 'REAL32') then - call hist_buffer_norm_value(buff_ptr, field_data_r4(:,1), logger=errors) - if (errors%num_errors() > 0) then - call errors%output(iulog) - write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' - call endrun(errmsg) - end if + field_data_r4(:,1) = field_data_r8(:,1) call cam_grid_write_dist_array(this%hist_files(split_file_index), field_decomp, (/dim_sizes(1)/), & field_shape, field_data_r4(:,1), varid) else - call hist_buffer_norm_value(buff_ptr, field_data_r8(:,1), logger=errors) - if (errors%num_errors() > 0) then - call errors%output(iulog) - write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' - call endrun(errmsg) - end if call cam_grid_write_dist_array(this%hist_files(split_file_index), field_decomp, (/dim_sizes(1)/), & field_shape, field_data_r8(:,1), varid) end if else + call hist_buffer_norm_value(buff_ptr, field_data_r8, logger=errors) + if (errors%num_errors() > 0) then + call errors%output(iulog) + write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' + call endrun(errmsg) + end if if (trim(field_precision) == 'REAL32') then - call hist_buffer_norm_value(buff_ptr, field_data_r4, logger=errors) - if (errors%num_errors() > 0) then - call errors%output(iulog) - write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' - call endrun(errmsg) - end if + field_data_r4 = field_data_r8 call cam_grid_write_dist_array(this%hist_files(split_file_index), field_decomp, dim_sizes(1:frank), & field_shape, field_data_r4, varid) else - call hist_buffer_norm_value(buff_ptr, field_data_r8, logger=errors) - call errors%output(iulog) - if (errors%num_errors() > 0) then - call errors%output(iulog) - write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' - call endrun(errmsg) - end if call cam_grid_write_dist_array(this%hist_files(split_file_index), field_decomp, dim_sizes(1:frank), & field_shape, field_data_r8, varid) end if From 8283239a4ec28277cf435e155b30c1a2d0e6f6f2 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Tue, 7 Apr 2026 15:44:19 -0600 Subject: [PATCH 08/15] overwrite accum fields to inst when outputting every timestep --- cime_config/hist_config.py | 30 +++++++++++++++++++ .../hist_config_files/atm_in_flat | 6 ++-- .../hist_config_files/user_nl_cam_flat | 4 ++- test/unit/python/test_hist_config.py | 2 +- 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/cime_config/hist_config.py b/cime_config/hist_config.py index 1be9ac485..ab5b5aede 100644 --- a/cime_config/hist_config.py +++ b/cime_config/hist_config.py @@ -409,6 +409,9 @@ def num_fields(self): """Return the number of fields in this HistFieldList object.""" return len(self.__field_names) + @property + def field_names(self): + return self.__field_names def output_nl_fieldlist(self, outfile, field_varname): """Output the field name of this HistFieldList object as a namelist @@ -685,6 +688,28 @@ def remove_fields(self, fields, pobj, logger): # end if return all_removed + def convert_accumulated_fields(self, pobj, logger): + """Move all accumulated fields to instantaneous field list""" + if self.__var_fields.num_fields() > 0: + self.__inst_fields.add_fields(self.__var_fields.field_names, pobj, logger) + self.__var_fields.remove_fields(self.__var_fields.field_names, pobj, logger) + # end if + + if self.__avg_fields.num_fields() > 0: + self.__inst_fields.add_fields(self.__avg_fields.field_names, pobj, logger) + self.__avg_fields.remove_fields(self.__avg_fields.field_names, pobj, logger) + # end if + + if self.__min_fields.num_fields() > 0: + self.__inst_fields.add_fields(self.__min_fields.field_names, pobj, logger) + self.__min_fields.remove_fields(self.__min_fields.field_names, pobj, logger) + # end if + + if self.__max_fields.num_fields() > 0: + self.__inst_fields.add_fields(self.__max_fields.field_names, pobj, logger) + self.__max_fields.remove_fields(self.__max_fields.field_names, pobj, logger) + # end if + @property def volume(self): """Return the volume for this HistoryVolConfig object""" @@ -794,6 +819,11 @@ def set_output_frequency(self, ofreq, pobj, logger): ctx = context_string(pobj) logger.debug(f"Setting output_frequency to '{ofreq}'{ctx}") # end if + # Convert accumulated fields to instantaneous fields if we're set + # to output every timestep + if self.__output_freq == (1, 'nsteps'): + self.convert_accumulated_fields(pobj, logger) + # end if return True # end if emsg = f"Attempt to set unrecognized output_frequency, '{ofreq}'" diff --git a/test/unit/python/sample_files/hist_config_files/atm_in_flat b/test/unit/python/sample_files/hist_config_files/atm_in_flat index 86efcd4bb..bd5cb4296 100644 --- a/test/unit/python/sample_files/hist_config_files/atm_in_flat +++ b/test/unit/python/sample_files/hist_config_files/atm_in_flat @@ -1,6 +1,6 @@ &hist_config_arrays_nl - hist_num_inst_fields = 3 + hist_num_inst_fields = 4 hist_num_avg_fields = 3 hist_num_min_fields = 3 hist_num_max_fields = 2 @@ -23,9 +23,9 @@ &hist_file_config_nl hist_volume = 'h3' - hist_inst_fields = 'T', 'U', 'V' + hist_inst_fields = 'T ', 'U ', 'V ', 'PS' hist_max_frames = 24 - hist_output_frequency = '2*nsteps' + hist_output_frequency = '1*nsteps' hist_precision = 'REAL64' hist_file_type = 'history' hist_filename_spec = '%c.cam.%u%f.%y-%m-%d-%s.nc' diff --git a/test/unit/python/sample_files/hist_config_files/user_nl_cam_flat b/test/unit/python/sample_files/hist_config_files/user_nl_cam_flat index a03e0c603..4d208ecc8 100644 --- a/test/unit/python/sample_files/hist_config_files/user_nl_cam_flat +++ b/test/unit/python/sample_files/hist_config_files/user_nl_cam_flat @@ -13,7 +13,9 @@ use_topo_file = .false. ! More history configuration hist_add_inst_fields;h3: T, U, V -hist_output_frequency;h3: 2*nsteps +! This avg field will be converted to inst because output freq = 1*nsteps +hist_add_avg_fields;h3: PS +hist_output_frequency;h3: 1*nsteps hist_precision;h3: REAL64 hist_max_frames;h3: 24 diff --git a/test/unit/python/test_hist_config.py b/test/unit/python/test_hist_config.py index 57fad85f7..d3a6d5f65 100644 --- a/test/unit/python/test_hist_config.py +++ b/test/unit/python/test_hist_config.py @@ -117,7 +117,7 @@ def test_flat_user_nl_cam(self): self._test_config(hconfig, 'h1', 'REAL32', 30, (14, 'hours'), 'history', '.false.', '%c.cam.%u%f.%y-%m-%d-%s.nc', '%c.cam.r%u.%y-%m-%d-%s.nc') self.assertTrue('h3' in hist_configs, msg="'h3' not in hist_configs") hconfig = hist_configs['h3'] - self._test_config(hconfig, 'h3', 'REAL64', 24, (2, 'nsteps'), 'history', '.false.', '%c.cam.%u%f.%y-%m-%d-%s.nc', '%c.cam.r%u.%y-%m-%d-%s.nc') + self._test_config(hconfig, 'h3', 'REAL64', 24, (1, 'nsteps'), 'history', '.false.', '%c.cam.%u%f.%y-%m-%d-%s.nc', '%c.cam.r%u.%y-%m-%d-%s.nc') _LOGGER.setLevel(logging.DEBUG) # Write out the namelist file with open(out_source, 'w', encoding='utf-8') as nl_file: From ad5b5a5a49711ce4b83c4b0127266835d6448eca Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Thu, 9 Apr 2026 13:09:18 -0600 Subject: [PATCH 09/15] remove duplicated info in hist objects --- .gitmodules | 2 +- src/history/buffers | 2 +- src/history/cam_hist_file.F90 | 11 ++++------- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.gitmodules b/.gitmodules index 0f8b90d4e..d5f19a935 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,7 +7,7 @@ [submodule "history"] path = src/history/buffers url = https://github.com/peverwhee/history_output - fxtag = b213964573e0e87b7a10acf627f1b435643a0b78 + fxtag = 700d1c5e462dc02e2cd711ccd632ce39df173990 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESMCI/history_output [submodule "mpas"] diff --git a/src/history/buffers b/src/history/buffers index b21396457..700d1c5e4 160000 --- a/src/history/buffers +++ b/src/history/buffers @@ -1 +1 @@ -Subproject commit b213964573e0e87b7a10acf627f1b435643a0b78 +Subproject commit 700d1c5e462dc02e2cd711ccd632ce39df173990 diff --git a/src/history/cam_hist_file.F90 b/src/history/cam_hist_file.F90 index 938b138ae..437936f5e 100644 --- a/src/history/cam_hist_file.F90 +++ b/src/history/cam_hist_file.F90 @@ -777,7 +777,7 @@ subroutine config_set_up_fields(this, possible_field_list) flag_xyfill=field_ptr%flag_xyfill(), & sampling_seq=field_ptr%sampling_sequence()) call hist_new_buffer(field_info, field_shape, & - this%rl_kind, 1, this%accumulate_types(idx), 1, errors=errors) + 1, this%accumulate_types(idx), 1, errors=errors) if (masterproc .and. errors%num_errors() > 0) then call errors%output(iulog) call endrun(subname//' error(s) during buffer creation') @@ -1513,8 +1513,7 @@ end subroutine config_write_time_dependent_variables subroutine config_write_field(this, field, split_file_index, restart, & sample_index, field_index, field_precision) use pio, only: PIO_OFFSET_KIND, pio_setframe - use hist_buffer, only: hist_buffer_t - use hist_api, only: hist_buffer_norm_value + use hist_api, only: hist_field_norm_value use cam_grid_support, only: cam_grid_write_dist_array use cam_abortutils, only: check_allocate, endrun use hist_field, only: hist_field_info_t @@ -1545,7 +1544,6 @@ subroutine config_write_field(this, field, split_file_index, restart, & integer :: idx real(r8), allocatable :: field_data_r8(:,:) real(r4), allocatable :: field_data_r4(:,:) - class(hist_buffer_t), pointer :: buff_ptr type(hist_log_messages) :: errors character(len=CL) :: errmsg character(len=*), parameter :: subname = 'config_write_field: ' @@ -1592,9 +1590,8 @@ subroutine config_write_field(this, field, split_file_index, restart, & do patch_idx = 1, num_patches varid = this%file_varids(field_index, patch_idx) call pio_setframe(this%hist_files(split_file_index), varid, int(sample_index,kind=PIO_OFFSET_KIND)) - buff_ptr => field%buffers if (frank == 1) then - call hist_buffer_norm_value(buff_ptr, field_data_r8(:,1), logger=errors) + call hist_field_norm_value(field, field_data_r8(:,1), logger=errors) if (errors%num_errors() > 0) then call errors%output(iulog) write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' @@ -1609,7 +1606,7 @@ subroutine config_write_field(this, field, split_file_index, restart, & field_shape, field_data_r8(:,1), varid) end if else - call hist_buffer_norm_value(buff_ptr, field_data_r8, logger=errors) + call hist_field_norm_value(field, field_data_r8, logger=errors) if (errors%num_errors() > 0) then call errors%output(iulog) write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' From 248735b934076a2735b22a6bd3be684deef59936 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Thu, 9 Apr 2026 15:53:04 -0600 Subject: [PATCH 10/15] fix bad field removal logic --- cime_config/hist_config.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cime_config/hist_config.py b/cime_config/hist_config.py index ab5b5aede..bcd6b53b3 100644 --- a/cime_config/hist_config.py +++ b/cime_config/hist_config.py @@ -409,10 +409,6 @@ def num_fields(self): """Return the number of fields in this HistFieldList object.""" return len(self.__field_names) - @property - def field_names(self): - return self.__field_names - def output_nl_fieldlist(self, outfile, field_varname): """Output the field name of this HistFieldList object as a namelist variable that is an array of strings. @@ -692,22 +688,26 @@ def convert_accumulated_fields(self, pobj, logger): """Move all accumulated fields to instantaneous field list""" if self.__var_fields.num_fields() > 0: self.__inst_fields.add_fields(self.__var_fields.field_names, pobj, logger) - self.__var_fields.remove_fields(self.__var_fields.field_names, pobj, logger) + var_field_list = list(self.__var_fields.field_names) + self.__var_fields.remove_fields(var_field_list, pobj, logger) # end if if self.__avg_fields.num_fields() > 0: self.__inst_fields.add_fields(self.__avg_fields.field_names, pobj, logger) - self.__avg_fields.remove_fields(self.__avg_fields.field_names, pobj, logger) + avg_field_list = list(self.__avg_fields.field_names) + self.__avg_fields.remove_fields(avg_field_list, pobj, logger) # end if if self.__min_fields.num_fields() > 0: self.__inst_fields.add_fields(self.__min_fields.field_names, pobj, logger) - self.__min_fields.remove_fields(self.__min_fields.field_names, pobj, logger) + min_field_list = list(self.__min_fields.field_names) + self.__min_fields.remove_fields(min_field_list.field_names, pobj, logger) # end if if self.__max_fields.num_fields() > 0: self.__inst_fields.add_fields(self.__max_fields.field_names, pobj, logger) - self.__max_fields.remove_fields(self.__max_fields.field_names, pobj, logger) + max_field_list = list(self.__max_fields.field_names) + self.__max_fields.remove_fields(max_field_list, pobj, logger) # end if @property From 6aa35a433af1f6a29dbf69c24ee642378427d348 Mon Sep 17 00:00:00 2001 From: peverwhee Date: Tue, 14 Apr 2026 10:58:05 -0600 Subject: [PATCH 11/15] fix bad sampling logic --- .gitmodules | 2 +- src/history/buffers | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index d5f19a935..35dcf55ab 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,7 +7,7 @@ [submodule "history"] path = src/history/buffers url = https://github.com/peverwhee/history_output - fxtag = 700d1c5e462dc02e2cd711ccd632ce39df173990 + fxtag = f9e32e2ad5b21bfcaca3efcf54b48c9b8fda39c9 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESMCI/history_output [submodule "mpas"] diff --git a/src/history/buffers b/src/history/buffers index 700d1c5e4..f9e32e2ad 160000 --- a/src/history/buffers +++ b/src/history/buffers @@ -1 +1 @@ -Subproject commit 700d1c5e462dc02e2cd711ccd632ce39df173990 +Subproject commit f9e32e2ad5b21bfcaca3efcf54b48c9b8fda39c9 From 5f977a7ba318ca18ec440ca4ab3e2e4c7045db9d Mon Sep 17 00:00:00 2001 From: peverwhee Date: Thu, 16 Apr 2026 11:15:47 -0600 Subject: [PATCH 12/15] add new history test --- cime_config/testdefs/testlist_cam.xml | 12 ++++++++++++ .../shell_commands | 1 + .../outfrq_kessler_mpas_derecho_history/user_nl_cam | 8 ++++++++ 3 files changed, 21 insertions(+) create mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq_kessler_mpas_derecho_history/shell_commands create mode 100644 cime_config/testdefs/testmods_dirs/cam/outfrq_kessler_mpas_derecho_history/user_nl_cam diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index 9512c0f95..3d7781b49 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -177,6 +177,18 @@ + + + + + + + + + + + + diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_kessler_mpas_derecho_history/shell_commands b/cime_config/testdefs/testmods_dirs/cam/outfrq_kessler_mpas_derecho_history/shell_commands new file mode 100644 index 000000000..fd81526e2 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_kessler_mpas_derecho_history/shell_commands @@ -0,0 +1 @@ + ./xmlchange CAM_LINKED_LIBS="-lmpas" diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_kessler_mpas_derecho_history/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq_kessler_mpas_derecho_history/user_nl_cam new file mode 100644 index 000000000..a0260f636 --- /dev/null +++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_kessler_mpas_derecho_history/user_nl_cam @@ -0,0 +1,8 @@ +debug_output=0 +hist_add_inst_fields;h3: Q,T,U,V,PS +hist_add_avg_fields;h3: PSDRY +hist_add_max_fields;h3: PHIS +hist_add_min_fields;h3: PDEL, PDELDRY +hist_add_var_fields;h3: ZM, PRECT +hist_output_frequency;h3: 2*nsteps +hist_write_nstep0;h3: .true. From 216abc61b3c5d7c6fff810e2844c836ddf47200e Mon Sep 17 00:00:00 2001 From: peverwhee Date: Mon, 11 May 2026 19:59:39 -0600 Subject: [PATCH 13/15] update history hash --- .gitmodules | 2 +- src/history/buffers | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 85a17276f..63b8e133c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,7 +7,7 @@ [submodule "history"] path = src/history/buffers url = https://github.com/peverwhee/history_output - fxtag = f9e32e2ad5b21bfcaca3efcf54b48c9b8fda39c9 + fxtag = 5db5aa177a1258933fafd2db217a219afd8a2bcd fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESMCI/history_output [submodule "mpas"] diff --git a/src/history/buffers b/src/history/buffers index f9e32e2ad..5db5aa177 160000 --- a/src/history/buffers +++ b/src/history/buffers @@ -1 +1 @@ -Subproject commit f9e32e2ad5b21bfcaca3efcf54b48c9b8fda39c9 +Subproject commit 5db5aa177a1258933fafd2db217a219afd8a2bcd From d064a552513c94609642aa0c3d4d9c7ca8b93648 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Thu, 14 May 2026 15:33:15 -0600 Subject: [PATCH 14/15] address review comments; error during preview_namelist when duplicate hist fields with different flags on same volume --- cime_config/hist_config.py | 50 +++++++++++++++++---------- cime_config/testdefs/testlist_cam.xml | 2 +- src/history/cam_hist_file.F90 | 37 ++++++++------------ test/unit/python/test_hist_config.py | 6 ++-- 4 files changed, 51 insertions(+), 44 deletions(-) diff --git a/cime_config/hist_config.py b/cime_config/hist_config.py index bcd6b53b3..ce5ea4a7d 100644 --- a/cime_config/hist_config.py +++ b/cime_config/hist_config.py @@ -345,23 +345,34 @@ def __init__(self, volume, list_type, list_desc): self.__field_names = [] self.__max_namelen = 0 - def _add_item(self, item, pobj, logger): + def _add_item(self, item, pobj, full_field_list, logger): """Add field name, to this object and return True if this field name was added. is a single item to be added. is the ParseObject source of . + is the list of all fields for this volume """ if not _is_string(item)[0]: errmsg = f"Bad diagnostic name, '{item}'" pobj.add_syntax_err(errmsg) return False # end if - if item in self.__field_names: - # Field is a duplicate - ctx = context_string(pobj) - logger.warning(f"Field, '{item}' already in {self.desc} fields for hist volume, {self.volume}{ctx}") - return False - # end if + for field_list in full_field_list: + if item in field_list.__field_names: + if self == field_list: + # Field is a duplicate, same flag (so this is just a warning) + ctx = context_string(pobj) + logger.warning(f"Field, '{item}' already in {self.desc} fields for hist volume, {self.volume}{ctx}") + return False + else: + # Field is a duplicate, different flag (this is an error) + ctx = context_string(pobj) + emsg = f"Field, '{item}' already in {field_list.desc} fields for hist volume, {self.volume}. Cannot also add to {self.desc} fields for the same volume{ctx}" + pobj.add_syntax_err(emsg) + return False + # end if + # end if + # end for self.__field_names.append(item) self.__max_namelen = max(len(item), self.__max_namelen) if logger.getEffectiveLevel() <= logging.DEBUG: @@ -370,18 +381,19 @@ def _add_item(self, item, pobj, logger): # end if return True - def add_fields(self, items, pobj, logger): + def add_fields(self, items, pobj, full_field_list, logger): """Add to this object and return True if all items were added. can be a single item or a list. is the ParseObject source of + is the list of all fields for this volume """ if isinstance(items, list): do_add = True for item in items: - do_add &= self._add_item(item, pobj, logger) + do_add &= self._add_item(item, pobj, full_field_list, logger) # end for else: - do_add = self._add_item(items, pobj, logger) + do_add = self._add_item(items, pobj, full_field_list, logger) # end if return do_add @@ -632,7 +644,7 @@ def add_inst_fields(self, fields, pobj, logger): HistoryVolConfig object. Return True if it was okay to add to list of last fields. """ - add_ok = self.__inst_fields.add_fields(fields, pobj, logger) + add_ok = self.__inst_fields.add_fields(fields, pobj, self.__all_fields, logger) return add_ok def add_avg_fields(self, fields, pobj, logger): @@ -640,28 +652,28 @@ def add_avg_fields(self, fields, pobj, logger): object. Return True if it was okay to add to list of avg fields. """ - add_ok = self.__avg_fields.add_fields(fields, pobj, logger) + add_ok = self.__avg_fields.add_fields(fields, pobj, self.__all_fields, logger) return add_ok def add_min_fields(self, fields, pobj, logger): """Add one or more min_fields to this HistoryVolConfig object. Return True if it was okay to add to list of min fields. """ - add_ok = self.__min_fields.add_fields(fields, pobj, logger) + add_ok = self.__min_fields.add_fields(fields, pobj, self.__all_fields, logger) return add_ok def add_max_fields(self, fields, pobj, logger): """Add one or more max_fields to this HistoryVolConfig object. Return True if it was okay to add to list of max fields. """ - add_ok = self.__max_fields.add_fields(fields, pobj, logger) + add_ok = self.__max_fields.add_fields(fields, pobj, self.__all_fields, logger) return add_ok def add_var_fields(self, fields, pobj, logger): """Add one or more var_fields to this HistoryVolConfig object. Return True if it was okay to add to list of var fields. """ - add_ok = self.__var_fields.add_fields(fields, pobj, logger) + add_ok = self.__var_fields.add_fields(fields, pobj, self.__all_fields, logger) return add_ok def remove_fields(self, fields, pobj, logger): @@ -687,25 +699,25 @@ def remove_fields(self, fields, pobj, logger): def convert_accumulated_fields(self, pobj, logger): """Move all accumulated fields to instantaneous field list""" if self.__var_fields.num_fields() > 0: - self.__inst_fields.add_fields(self.__var_fields.field_names, pobj, logger) + self.__inst_fields.add_fields(self.__var_fields.field_names, pobj, [self.__inst_fields], logger) var_field_list = list(self.__var_fields.field_names) self.__var_fields.remove_fields(var_field_list, pobj, logger) # end if if self.__avg_fields.num_fields() > 0: - self.__inst_fields.add_fields(self.__avg_fields.field_names, pobj, logger) + self.__inst_fields.add_fields(self.__avg_fields.field_names, pobj, [self.__inst_fields], logger) avg_field_list = list(self.__avg_fields.field_names) self.__avg_fields.remove_fields(avg_field_list, pobj, logger) # end if if self.__min_fields.num_fields() > 0: - self.__inst_fields.add_fields(self.__min_fields.field_names, pobj, logger) + self.__inst_fields.add_fields(self.__min_fields.field_names, pobj, [self.__inst_fields], logger) min_field_list = list(self.__min_fields.field_names) self.__min_fields.remove_fields(min_field_list.field_names, pobj, logger) # end if if self.__max_fields.num_fields() > 0: - self.__inst_fields.add_fields(self.__max_fields.field_names, pobj, logger) + self.__inst_fields.add_fields(self.__max_fields.field_names, pobj, [self.__inst_fields], logger) max_field_list = list(self.__max_fields.field_names) self.__max_fields.remove_fields(max_field_list, pobj, logger) # end if diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml index 985515dfc..5b1f8c55d 100644 --- a/cime_config/testdefs/testlist_cam.xml +++ b/cime_config/testdefs/testlist_cam.xml @@ -178,7 +178,7 @@ - + diff --git a/src/history/cam_hist_file.F90 b/src/history/cam_hist_file.F90 index 437936f5e..afad4316c 100644 --- a/src/history/cam_hist_file.F90 +++ b/src/history/cam_hist_file.F90 @@ -1542,8 +1542,7 @@ subroutine config_write_field(this, field, split_file_index, restart, & type(var_desc_t) :: varid integer :: field_decomp integer :: idx - real(r8), allocatable :: field_data_r8(:,:) - real(r4), allocatable :: field_data_r4(:,:) + real(r8), allocatable :: field_data(:,:) type(hist_log_messages) :: errors character(len=CL) :: errmsg character(len=*), parameter :: subname = 'config_write_field: ' @@ -1556,23 +1555,19 @@ subroutine config_write_field(this, field, split_file_index, restart, & frank = size(field_shape) if (frank == 1) then if (trim(field_precision) == 'REAL32') then - allocate(field_data_r4(end_dims(1) - beg_dims(1) + 1, 1), stat=ierr, errmsg=errmsg) - call check_allocate(ierr, subname, 'field_data_r4', file=__FILE__, line=__LINE__-1, errmsg=errmsg) - allocate(field_data_r8(end_dims(1) - beg_dims(1) + 1, 1), stat=ierr, errmsg=errmsg) - call check_allocate(ierr, subname, 'field_data_r8', file=__FILE__, line=__LINE__-1, errmsg=errmsg) + allocate(field_data(end_dims(1) - beg_dims(1) + 1, 1), stat=ierr, errmsg=errmsg) + call check_allocate(ierr, subname, 'field_data', file=__FILE__, line=__LINE__-1, errmsg=errmsg) else - allocate(field_data_r8(end_dims(1) - beg_dims(1) + 1, 1), stat=ierr, errmsg=errmsg) - call check_allocate(ierr, subname, 'field_data_r8', file=__FILE__, line=__LINE__-1, errmsg=errmsg) + allocate(field_data(end_dims(1) - beg_dims(1) + 1, 1), stat=ierr, errmsg=errmsg) + call check_allocate(ierr, subname, 'field_data', file=__FILE__, line=__LINE__-1, errmsg=errmsg) end if else if (trim(field_precision) == 'REAL32') then - allocate(field_data_r4(end_dims(1) - beg_dims(1) + 1, field_shape(2)), stat=ierr, errmsg=errmsg) - call check_allocate(ierr, subname, 'field_data_r4', file=__FILE__, line=__LINE__-1, errmsg=errmsg) - allocate(field_data_r8(end_dims(1) - beg_dims(1) + 1, field_shape(2)), stat=ierr, errmsg=errmsg) - call check_allocate(ierr, subname, 'field_data_r8', file=__FILE__, line=__LINE__-1, errmsg=errmsg) + allocate(field_data(end_dims(1) - beg_dims(1) + 1, field_shape(2)), stat=ierr, errmsg=errmsg) + call check_allocate(ierr, subname, 'field_data', file=__FILE__, line=__LINE__-1, errmsg=errmsg) else - allocate(field_data_r8(end_dims(1) - beg_dims(1) + 1, field_shape(2)), stat=ierr, errmsg=errmsg) - call check_allocate(ierr, subname, 'field_data_r8', file=__FILE__, line=__LINE__-1, errmsg=errmsg) + allocate(field_data(end_dims(1) - beg_dims(1) + 1, field_shape(2)), stat=ierr, errmsg=errmsg) + call check_allocate(ierr, subname, 'field_data', file=__FILE__, line=__LINE__-1, errmsg=errmsg) end if end if ! Shape of array @@ -1591,34 +1586,32 @@ subroutine config_write_field(this, field, split_file_index, restart, & varid = this%file_varids(field_index, patch_idx) call pio_setframe(this%hist_files(split_file_index), varid, int(sample_index,kind=PIO_OFFSET_KIND)) if (frank == 1) then - call hist_field_norm_value(field, field_data_r8(:,1), logger=errors) + call hist_field_norm_value(field, field_data(:,1), logger=errors) if (errors%num_errors() > 0) then call errors%output(iulog) write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' call endrun(errmsg) end if if (trim(field_precision) == 'REAL32') then - field_data_r4(:,1) = field_data_r8(:,1) call cam_grid_write_dist_array(this%hist_files(split_file_index), field_decomp, (/dim_sizes(1)/), & - field_shape, field_data_r4(:,1), varid) + field_shape, real(field_data(:,1), r4), varid) else call cam_grid_write_dist_array(this%hist_files(split_file_index), field_decomp, (/dim_sizes(1)/), & - field_shape, field_data_r8(:,1), varid) + field_shape, field_data(:,1), varid) end if else - call hist_field_norm_value(field, field_data_r8, logger=errors) + call hist_field_norm_value(field, field_data, logger=errors) if (errors%num_errors() > 0) then call errors%output(iulog) write(errmsg, *) subname, 'ERROR writing field "', trim(field%diag_name()), '"' call endrun(errmsg) end if if (trim(field_precision) == 'REAL32') then - field_data_r4 = field_data_r8 call cam_grid_write_dist_array(this%hist_files(split_file_index), field_decomp, dim_sizes(1:frank), & - field_shape, field_data_r4, varid) + field_shape, real(field_data, r4), varid) else call cam_grid_write_dist_array(this%hist_files(split_file_index), field_decomp, dim_sizes(1:frank), & - field_shape, field_data_r8, varid) + field_shape, field_data, varid) end if end if end do diff --git a/test/unit/python/test_hist_config.py b/test/unit/python/test_hist_config.py index d3a6d5f65..7a1b0b42f 100644 --- a/test/unit/python/test_hist_config.py +++ b/test/unit/python/test_hist_config.py @@ -188,13 +188,14 @@ def test_bad_user_nl_cam(self): # Read in file: file_lines = old_file.readlines() # Edit to add bad lines - file_lines[8] = "" + file_lines[8] = "hist_add_inst_fields;h3: T\n" file_lines[9] = "hist_remove_fields;h0:\n" file_lines[10] = "hist_output_frequency;h0: 1+nmonths\n" file_lines[11] = "hist_precision;h0: REAL34\n" file_lines[13] = "hist_add_inst_fields;h3: T&U&V\n" file_lines[16] = "hist_max_frames;h3: -24\n" file_lines[17] = "hist_write_nstep0;h3: treu\n" + file_lines[18] = "hist_add_avg_fields;h3: T\n" # end with # Create a new modified version of the file with the bad entries @@ -215,7 +216,8 @@ def test_bad_user_nl_cam(self): "Found invalid identifiers", "T&U&V, at", "Attempt to set max frames to '-24', must be a positive integer, at", - "hist_write_nstep0 must be one of .false., .true., f, false, t, true, at"] + "hist_write_nstep0 must be one of .false., .true., f, false, t, true, at", + "Field, 'T' already in instantaneous fields for hist volume, h3. Cannot also add to average fields for the same volume, at"] # Check error messages are as expected for index, errmsg in enumerate(exception_split): From 1fea7da11b754af9094288bc33bc01e389fc5a18 Mon Sep 17 00:00:00 2001 From: Courtney Peverley Date: Thu, 14 May 2026 15:41:35 -0600 Subject: [PATCH 15/15] use history tag --- .gitmodules | 14 +++++++------- src/history/buffers | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitmodules b/.gitmodules index f6baee2a1..35855cd4b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,9 +5,9 @@ fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework [submodule "history"] - path = src/history/buffers - url = https://github.com/peverwhee/history_output - fxtag = 5db5aa177a1258933fafd2db217a219afd8a2bcd + path = src/history/buffers + url = https://github.com/ESMCI/history_output + fxtag = history01_01 fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESMCI/history_output [submodule "mpas"] @@ -24,8 +24,8 @@ fxrequired = AlwaysRequired fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics [submodule "rrtmgp-data"] - path = src/physics/utils/rrtmgp-data - url = https://github.com/earth-system-radiation/rrtmgp-data.git + path = src/physics/utils/rrtmgp-data + url = https://github.com/earth-system-radiation/rrtmgp-data.git fxrequired = AlwaysRequired fxtag = v1.8 fxDONOTUSEurl = https://github.com/earth-system-radiation/rrtmgp-data.git @@ -102,8 +102,8 @@ fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/ESCOMP/CESM_share [submodule "tools/CUPiD"] - path = tools/CUPiD - url = https://github.com/NCAR/CUPiD.git + path = tools/CUPiD + url = https://github.com/NCAR/CUPiD.git fxtag = v0.4.0 fxrequired = ToplevelRequired fxDONOTUSEurl = https://github.com/NCAR/CUPiD.git diff --git a/src/history/buffers b/src/history/buffers index 5db5aa177..dc000fe06 160000 --- a/src/history/buffers +++ b/src/history/buffers @@ -1 +1 @@ -Subproject commit 5db5aa177a1258933fafd2db217a219afd8a2bcd +Subproject commit dc000fe064e10e60baa9263fbc60eccd1bc84309