Add support for accumulated history fields

.gitmodules

@@ -5,9 +5,9 @@
         fxrequired    = AlwaysRequired
 	fxDONOTUSEurl = https://github.com/NCAR/ccpp-framework
 [submodule "history"]
-	path = src/history/buffers
-	url = https://github.com/ESMCI/history_output
-	fxtag         = history01_00
+	path          = src/history/buffers
+	url           = https://github.com/ESMCI/history_output
+	fxtag         = history01_01
 	fxrequired    = AlwaysRequired
 	fxDONOTUSEurl = https://github.com/ESMCI/history_output
 [submodule "mpas"]
@@ -24,8 +24,8 @@
         fxrequired    = AlwaysRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
 [submodule "rrtmgp-data"]
-	path = src/physics/utils/rrtmgp-data
-	url = https://github.com/earth-system-radiation/rrtmgp-data.git
+	path          = src/physics/utils/rrtmgp-data
+	url           = https://github.com/earth-system-radiation/rrtmgp-data.git
         fxrequired    = AlwaysRequired
         fxtag         = v1.8
         fxDONOTUSEurl = https://github.com/earth-system-radiation/rrtmgp-data.git
@@ -102,8 +102,8 @@
         fxrequired    = ToplevelRequired
 	fxDONOTUSEurl = https://github.com/ESCOMP/CESM_share
 [submodule "tools/CUPiD"]
-	path = tools/CUPiD
-	url = https://github.com/NCAR/CUPiD.git
+	path          = tools/CUPiD
+	url           = https://github.com/NCAR/CUPiD.git
 	fxtag         = v0.4.0
 	fxrequired    = ToplevelRequired
 	fxDONOTUSEurl = https://github.com/NCAR/CUPiD.git

cime_config/atm_in_paramgen.py

@@ -1410,7 +1410,7 @@ def append_user_nl_file(self, user_nl_file):
                         #End if (array indices)
 
                         #Extract value string:
-                        val_str   = ' '.join(line_ss[1:]) # the rest is the value string
+                        val_str   = ' '.join(line_ss[1:]).strip() # the rest is the value string
 
                         #Check if value string ends in array continuation:
                         if is_array:

cime_config/hist_config.py

@@ -345,23 +345,34 @@ def __init__(self, volume, list_type, list_desc):
         self.__field_names = []
         self.__max_namelen = 0
 
-    def _add_item(self, item, pobj, logger):
+    def _add_item(self, item, pobj, full_field_list, logger):
         """Add field name, <item> to this object and return True if this
            field name was added.
         <item> is a single item to be added.
         <pobj> is the ParseObject source of <items>.
+        <full_field_list> is the list of all fields for this volume
         """
         if not _is_string(item)[0]:
             errmsg = f"Bad diagnostic name, '{item}'"
             pobj.add_syntax_err(errmsg)
             return False
         # end if
-        if item in self.__field_names:
-            # Field is a duplicate
-            ctx = context_string(pobj)
-            logger.warning(f"Field, '{item}' already in {self.desc} fields for hist volume, {self.volume}{ctx}")
-            return False
-        # end if
+        for field_list in full_field_list:
+            if item in field_list.__field_names:
+                if self == field_list:
+                    # Field is a duplicate, same flag (so this is just a warning)
+                    ctx = context_string(pobj)
+                    logger.warning(f"Field, '{item}' already in {self.desc} fields for hist volume, {self.volume}{ctx}")
+                    return False
+                else:
+                    # Field is a duplicate, different flag (this is an error)
+                    ctx = context_string(pobj)
+                    emsg = f"Field, '{item}' already in {field_list.desc} fields for hist volume, {self.volume}. Cannot also add to {self.desc} fields for the same volume{ctx}"
+                    pobj.add_syntax_err(emsg)
+                    return False
+                # end if
+            # end if
+        # end for
         self.__field_names.append(item)
         self.__max_namelen = max(len(item), self.__max_namelen)
         if logger.getEffectiveLevel() <= logging.DEBUG:
@@ -370,18 +381,19 @@ def _add_item(self, item, pobj, logger):
         # end if
         return True
 
-    def add_fields(self, items, pobj, logger):
+    def add_fields(self, items, pobj, full_field_list, logger):
         """Add <items> to this object and return True if all items were added.
         <items> can be a single item or a list.
         <pobj> is the ParseObject source of <items>
+        <full_field_list> is the list of all fields for this volume
         """
         if isinstance(items, list):
             do_add = True
             for item in items:
-                do_add &= self._add_item(item, pobj, logger)
+                do_add &= self._add_item(item, pobj, full_field_list, logger)
             # end for
         else:
-            do_add = self._add_item(items, pobj, logger)
+            do_add = self._add_item(items, pobj, full_field_list, logger)
         # end if
         return do_add
 
@@ -409,7 +421,6 @@ def num_fields(self):
         """Return the number of fields in this HistFieldList object."""
         return len(self.__field_names)
 
-
     def output_nl_fieldlist(self, outfile, field_varname):
         """Output the field name of this HistFieldList object as a namelist
         variable that is an array of strings.
@@ -633,36 +644,36 @@ def add_inst_fields(self, fields, pobj, logger):
            HistoryVolConfig object.
         Return True if it was okay to add <fields> to list of last fields.
         """
-        add_ok = self.__inst_fields.add_fields(fields, pobj, logger)
+        add_ok = self.__inst_fields.add_fields(fields, pobj, self.__all_fields, logger)
         return add_ok
 
     def add_avg_fields(self, fields, pobj, logger):
         """Add one or more time-averaged fields to this HistoryVolConfig
         object.
         Return True if it was okay to add <fields> to list of avg fields.
         """
-        add_ok = self.__avg_fields.add_fields(fields, pobj, logger)
+        add_ok = self.__avg_fields.add_fields(fields, pobj, self.__all_fields, logger)
         return add_ok
 
     def add_min_fields(self, fields, pobj, logger):
         """Add one or more min_fields to this HistoryVolConfig object.
         Return True if it was okay to add <fields> to list of min fields.
         """
-        add_ok = self.__min_fields.add_fields(fields, pobj, logger)
+        add_ok = self.__min_fields.add_fields(fields, pobj, self.__all_fields, logger)
         return add_ok
 
     def add_max_fields(self, fields, pobj, logger):
         """Add one or more max_fields to this HistoryVolConfig object.
         Return True if it was okay to add <fields> to list of max fields.
         """
-        add_ok = self.__max_fields.add_fields(fields, pobj, logger)
+        add_ok = self.__max_fields.add_fields(fields, pobj, self.__all_fields, logger)
         return add_ok
 
     def add_var_fields(self, fields, pobj, logger):
         """Add one or more var_fields to this HistoryVolConfig object.
         Return True if it was okay to add <fields> to list of var fields.
         """
-        add_ok = self.__var_fields.add_fields(fields, pobj, logger)
+        add_ok = self.__var_fields.add_fields(fields, pobj, self.__all_fields, logger)
         return add_ok
 
     def remove_fields(self, fields, pobj, logger):
@@ -685,6 +696,32 @@ def remove_fields(self, fields, pobj, logger):
         # end if
         return all_removed
 
+    def convert_accumulated_fields(self, pobj, logger):
+        """Move all accumulated fields to instantaneous field list"""
+        if self.__var_fields.num_fields() > 0:
+            self.__inst_fields.add_fields(self.__var_fields.field_names, pobj, [self.__inst_fields], logger)
+            var_field_list = list(self.__var_fields.field_names)
+            self.__var_fields.remove_fields(var_field_list, pobj, logger)
+        # end if
+
+        if self.__avg_fields.num_fields() > 0:
+            self.__inst_fields.add_fields(self.__avg_fields.field_names, pobj, [self.__inst_fields], logger)
+            avg_field_list = list(self.__avg_fields.field_names)
+            self.__avg_fields.remove_fields(avg_field_list, pobj, logger)
+        # end if
+
+        if self.__min_fields.num_fields() > 0:
+            self.__inst_fields.add_fields(self.__min_fields.field_names, pobj, [self.__inst_fields], logger)
+            min_field_list = list(self.__min_fields.field_names)
+            self.__min_fields.remove_fields(min_field_list.field_names, pobj, logger)
+        # end if
+
+        if self.__max_fields.num_fields() > 0:
+            self.__inst_fields.add_fields(self.__max_fields.field_names, pobj, [self.__inst_fields], logger)
+            max_field_list = list(self.__max_fields.field_names)
+            self.__max_fields.remove_fields(max_field_list, pobj, logger)
+        # end if
+
     @property
     def volume(self):
         """Return the volume for this HistoryVolConfig object"""
@@ -794,6 +831,11 @@ def set_output_frequency(self, ofreq, pobj, logger):
                 ctx = context_string(pobj)
                 logger.debug(f"Setting output_frequency to '{ofreq}'{ctx}")
             # end if
+            # Convert accumulated fields to instantaneous fields if we're set
+            # to output every timestep
+            if self.__output_freq == (1, 'nsteps'):
+                self.convert_accumulated_fields(pobj, logger)
+            # end if
             return True
         # end if
         emsg = f"Attempt to set unrecognized output_frequency, '{ofreq}'"

cime_config/testdefs/testlist_cam.xml

@@ -177,6 +177,18 @@
     </options>
   </test>
 
+  <!-- History output test -->
+  <test compset="FKESSLER" grid="mpasa480_mpasa480" name="SMS_Ln9" testmods="cam/outfrq_kessler_mpas_derecho_history">
+    <machines>
+      <machine name="derecho" compiler="intel" category="aux_sima"/>
+      <machine name="derecho" compiler="gnu" category="aux_sima"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:10:00</option>
+      <option name="comment">Test for both instantaneous and accumulated output.</option>
+    </options>
+  </test>
+
   <!-- Multiple history tape test -->
   <test compset="FKESSLER" grid="ne3pg3_ne3pg3_mg37" name="SMS_Ln9" testmods="cam/outfrq_se_cslam_multitape">
     <machines>

cime_config/testdefs/testmods_dirs/cam/outfrq_kessler_mpas_derecho_history/shell_commands

@@ -0,0 +1 @@
+ ./xmlchange CAM_LINKED_LIBS="-lmpas"

cime_config/testdefs/testmods_dirs/cam/outfrq_kessler_mpas_derecho_history/user_nl_cam

@@ -0,0 +1,8 @@
+debug_output=0
+hist_add_inst_fields;h3: Q,T,U,V,PS
+hist_add_avg_fields;h3: PSDRY
+hist_add_max_fields;h3: PHIS
+hist_add_min_fields;h3: PDEL, PDELDRY
+hist_add_var_fields;h3: ZM, PRECT
+hist_output_frequency;h3: 2*nsteps
+hist_write_nstep0;h3: .true.