Auto PR - develop → MAPL-v3 - added more 2D outputs to sample HISTORY.rc

CHANGELOG.md

@@ -13,18 +13,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Added optional SLURM "constraint".
 - Added functionality to run on tile space of stretched cube-sphere grids.
-- Added python package for post-processing ObsFcstAna output.
+- Added (and later revised) python package for post-processing ObsFcstAna output.
 
 
 ### Changed
 
+- Revised processing of HISTORY template (GEOSldas_HIST.rc).
 - Switch to using EASE grid tools in MAPL.
 - Specify only ntasks_model for SLURM resource request.
 - Revisions for handling of Nens and special nml and mwtrm path/files in coupled land-atm DAS.
 - Updated some defaults in LDASsa_DEFAULT_inputs_*.nml files.
 
 ### Fixed
 
+- Fixed ncra path for monthly compression
 - Fixed error from MAPL's ApplicationSupport.F90 to init UDUNITS.
 
 ### Removed

GEOSldas_App/GEOSldas_HIST.rc

@@ -1,24 +1,23 @@
 # Sample HISTORY.rc file for GEOSldas
 #
 # This HISTORY template is edited by "ldas_setup" via "process_hist.csh".  
-# The strings '#ASSIM', '#EASE', and '#CUBE' are *not* linked to MAPL HISTORY 
-# functionality.  For example, the line 
-#    "#CUBE            'tavg24_2d_lnd_Nx'"
-#  does *not* mean that the 'lnd' output will be on a cube-sphere grid. 
 
-#CUBE VERSION: 1
+VERSION: 1
+
+# Must edit 'EXPID' manually if HISTORY file is re-used without going through "ldas_setup".
 
-# Must edit 'EXPID' manually if HISTORY file is re-used without going  
-#  through "ldas_setup".
-#
 EXPID:  GEOSldas_expid
 
+# ------------------------------------------------------------------------------------------------
+
+# pre-defined Collections
+
 COLLECTIONS:
-#EASE            'tavg24_1d_lfs_Nt'
-#CUBE            'tavg24_2d_lfs_Nx'	
-#EASE            'tavg24_1d_lnd_Nt'	
-#CUBE            'tavg24_2d_lnd_Nx'
-#ASSIM           'SMAP_L4_SM_gph'
+#OUT1d           'tavg24_1d_lfs_Nt'
+#OUT2d           'tavg24_2d_lfs_Nx'
+#OUT1d           'tavg24_1d_lnd_Nt'
+#OUT2d           'tavg24_2d_lnd_Nx'
+#                'SMAP_L4_SM_gph'
 #                'inst1_1d_lnr_Nt'
 #                'catch_progn_incr'
 #                'inst3_1d_lndfcstana_Nt'
@@ -29,24 +28,34 @@ COLLECTIONS:
 #                'tavg24_1d_glc_Nt'
            ::
 
-#CUBE GRID_LABELS: PC720x361-DC
-#CUBE              PC1440x721-DC
+# --------------------------------------------------------------------------------------------------
+
+# 2d output can be on the following grids (see [COLLECTION_NAME].grid_label])
+
+GRID_LABELS: PC720x361-DC
+             PC1440x721-DC
+             EASEv2_M36
+      ::
+
+PC720x361-DC.GRID_TYPE:  LatLon
+PC720x361-DC.IM_WORLD:   720
+PC720x361-DC.JM_WORLD:   361
+PC720x361-DC.POLE:       PC
+PC720x361-DC.DATELINE:   DC
+PC720x361-DC.LM:         1
 
-#CUBE      ::
+PC1440x721-DC.GRID_TYPE: LatLon
+PC1440x721-DC.IM_WORLD:  1440
+PC1440x721-DC.JM_WORLD:   721
+PC1440x721-DC.POLE:      PC
+PC1440x721-DC.DATELINE:  DC
+PC1440x721-DC.LM:        1
 
-#CUBE PC720x361-DC.GRID_TYPE: LatLon
-#CUBE PC720x361-DC.IM_WORLD: 720
-#CUBE PC720x361-DC.JM_WORLD: 361
-#CUBE PC720x361-DC.POLE: PC
-#CUBE PC720x361-DC.DATELINE: DC
-#CUBE PC720x361-DC.LM: 1
+EASEv2_M36.GRID_TYPE:    EASE
+EASEv2_M36.GRIDNAME:     EASEv2_M36
+EASEv2_M36.LM:           1
 
-#CUBE PC1440x721-DC.GRID_TYPE: LatLon
-#CUBE PC1440x721-DC.IM_WORLD: 1440
-#CUBE PC1440x721-DC.JM_WORLD: 721
-#CUBE PC1440x721-DC.POLE: PC
-#CUBE PC1440x721-DC.DATELINE: DC
-#CUBE PC1440x721-DC.LM: 1
+# --------------------------------------------------------------------------------------------------
 
 # Detailed definition of the collections listed above 
 #
@@ -219,15 +228,16 @@ COLLECTIONS:
 
   tavg24_2d_lnd_Nx.format:        'CFIO',
   tavg24_2d_lnd_Nx.descr:         '2d,Daily,Time-Averaged,Single-Level,Assimilation,Land Surface Diagnostics',
-  tavg24_2d_lnd_Nx.nbits:        12,
+  tavg24_2d_lnd_Nx.nbits:         12,
   tavg24_2d_lnd_Nx.template:      '%y4%m2%d2_%h2%n2z.nc4',
   tavg24_2d_lnd_Nx.mode:          'time-averaged',
   tavg24_2d_lnd_Nx.frequency:     240000,
   tavg24_2d_lnd_Nx.ref_time:      000000,
-  tavg24_2d_lnd_Nx.regrid_exch:   '../input/tile.data'
-  tavg24_2d_lnd_Nx.regrid_name:   'GRIDNAME'
+  tavg24_2d_lnd_Nx.regrid_exch:   '../input/tile.data',
+  tavg24_2d_lnd_Nx.regrid_name:   'GRIDNAME',
 #  tavg24_2d_lnd_Nx.regrid_method: 'BILINEAR_MONOTONIC' ,                           
-  tavg24_2d_lnd_Nx.grid_label:    PC720x361-DC
+  tavg24_2d_lnd_Nx.grid_label:    PC720x361-DC,
+#  tavg24_2d_lnd_Nx.grid_label:   EASEv2_M36,
   tavg24_2d_lnd_Nx.deflate:       2,
   tavg24_2d_lnd_Nx.fields:     'GRN'        , 'VEGDYN'  ,
                                'LAI'        , 'VEGDYN'  ,

GEOSldas_App/ldas_setup

@@ -1283,12 +1283,18 @@ class LDASsetup:
                     shutil.copy2(histrc_file,tmprcfile)
                 else :
                     shutil.copy2(histrc_file,tmprcfile)
-                    GRID='EASE ' + self.rqdExeInp['GRIDNAME'] + ' ' +tmprcfile
-                    if '-CF' in self.rqdExeInp['GRIDNAME'] :
-                        GRID ='CUBE ' + self.rqdExeInp['GRIDNAME'] + ' ' +tmprcfile
-                    _assim = '1' if self.assim else '0'
-                    cmd =self.bindir +'/process_hist.csh '+ str(self.rqdExeInp['LSM_CHOICE']) + ' ' + str(self.rqdExeInp['AEROSOL_DEPOSITION']) + \
-                          ' ' + GRID + ' ' + str(self.rqdExeInp['RUN_IRRIG']) + ' ' + _assim + ' '+ str(self.nens)
+                    if 'EASE' in self.rqdExeInp['GRIDNAME'] :
+                        TMPSTR='OUT1d'
+                    else :
+                        TMPSTR='OUT2d'
+                    cmd = self.bindir +'/process_hist.csh'          + ' ' \
+                        + tmprcfile                                 + ' ' \
+                        + TMPSTR                                    + ' ' \
+                        + self.rqdExeInp['GRIDNAME']                + ' ' \
+                        + str(self.rqdExeInp['LSM_CHOICE'])         + ' ' \
+                        + str(self.rqdExeInp['AEROSOL_DEPOSITION']) + ' ' \
+                        + str(self.rqdExeInp['RUN_IRRIG'])          + ' ' \
+                        + str(self.nens)
                     print(cmd)
                     #os.system(cmd)
                     sp.call(shlex.split(cmd))

GEOSldas_App/lenkf_j_template.py

@@ -646,7 +646,7 @@
           if($NAVAIL != $NDAYS) continue
 
           # create monthly-mean nc4 file
-          ncra -h $EXPID.$ThisCol.${{YYYY}}${{MM}}*.nc4 ${{EXPID}}.${{ThisCol}}.monthly.$YYYY$MM.nc4
+          $BASEBIN/ncra -h $EXPID.$ThisCol.${{YYYY}}${{MM}}*.nc4 ${{EXPID}}.${{ThisCol}}.monthly.$YYYY$MM.nc4
 
           if($POSTPROC_HIST == 2) then
              # rudimentary check for desired nc4 file;  if ok, delete daily files

GEOSldas_App/process_hist.csh

@@ -1,56 +1,74 @@
 #!/bin/csh -f
 
-## I am changed the CUBE/EASE logic
-## if CUBE we produce 2D
-## anything else, SMAP and other offline grids we produce tile space
-
-setenv    LSM_CHOICE $1
-setenv    AEROSOL_DEPOSITION $2
-setenv    GRID $3
-setenv    GRIDNAME $4
-setenv    HISTRC $5
-setenv    RUN_IRRIG $6
-setenv    ASSIM  $7
-setenv    NENS $8
+# process GEOSldas_HIST.rc (=$HISTRC) template
+#
+# - turn on/off HIST collections depending on tile space
+#   - EASE:      turn on tile-space (1d) output
+#   - otherwise: turn on gridded    (2d) output
+# - turn on/off output variables depending on LSM_CHOICE, AEROSOL_DEPOSITION, and RUN_IRRIG
+# - fill in source 'GridComp' info for variables depending on NENS
+
+# process command line args
+
+setenv HISTRC             $1         # file name of HIST rc template (GEOSldas_HIST.rc)
+setenv OUTxd              $2         # "OUT1d" or "OUT2d" (to turn on/off collections)
+setenv GRIDNAME         "'$3'"       # full name of grid associated with tile space
+setenv LSM_CHOICE         $4
+setenv AEROSOL_DEPOSITION $5
+setenv RUN_IRRIG          $6
+setenv NENS               $7
+
+# -------------------------------------------------
 
 echo $GRIDNAME
 
-if($ASSIM == 1) then
-   sed -i 's|\#ASSIM|''|g' $HISTRC
-   sed -i '/^\#EASE/d' $HISTRC
-   sed -i '/^\#CUBE/d' $HISTRC
-else
-   sed -i '/^\#ASSIM/d' $HISTRC
-endif
+# uncomment 2d or 1d collections, depending on "OUT1d" (EASE tile space) or "OUT2d" (non-EASE tile space)
 
-if($GRID == CUBE) then
-   sed -i '/^\#EASE/d' $HISTRC
-   sed -i 's|\#CUBE|''|g' $HISTRC
-   sed -i 's|GRIDNAME|'"$GRIDNAME"'|g' $HISTRC
+if($OUTxd == OUT1d) then
+   sed -i 's|\#OUT1d|''|g' $HISTRC
 else
-   sed -i '/^\#CUBE/d' $HISTRC
-   sed -i 's|\#EASE|''|g' $HISTRC
-   sed -i 's|GRIDNAME|'"$GRIDNAME"'|g' $HISTRC
+   sed -i 's|\#OUT2d|''|g' $HISTRC
 endif
 
+# fill in name of grid associated with tile space
+
+sed -i -e  s/\'GRIDNAME\'/$GRIDNAME/g $HISTRC
+
+# set 'GridComp' based on LSM_CHOICE;
+# turn on/off variables associated with CATCHCN, AEROSOL_DEPOSITION, RUN_IRRIG
+
 if($LSM_CHOICE == 1) then
    set GridComp = CATCH
-   sed -i '/^>>>HIST_CATCHCN<<</d' $HISTRC
-   sed -i '/^>>>HIST_CATCHCNCLM45<<</d' $HISTRC
+   sed -i '/^>>>HIST_CATCHCN<<</d'         $HISTRC
+   sed -i '/^>>>HIST_CATCHCNCLM45<<</d'    $HISTRC
 endif
 
 if($LSM_CHOICE == 2) then
    set GridComp = CATCHCN
-   sed -i '/^>>>HIST_CATCHCNCLM45<<</d' $HISTRC
-   sed -i 's/>>>HIST_CATCHCN<<</''/g' $HISTRC
+   sed -i '/^>>>HIST_CATCHCNCLM45<<</d'    $HISTRC
+   sed -i 's/>>>HIST_CATCHCN<<</''/g'      $HISTRC
 endif
 
 if($LSM_CHOICE == 3) then
    set GridComp = CATCHCN
-   sed -i 's/>>>HIST_CATCHCN<<</''/g' $HISTRC
+   sed -i 's/>>>HIST_CATCHCN<<</''/g'      $HISTRC
    sed -i 's/>>>HIST_CATCHCNCLM45<<</''/g' $HISTRC
 endif
 
+if($AEROSOL_DEPOSITION == 0) then
+   sed -i '/^>>>HIST_AEROSOL<<</d'         $HISTRC
+else
+   sed -i 's/>>>HIST_AEROSOL<<</''/g'      $HISTRC
+endif
+
+if($RUN_IRRIG == 0) then
+   sed -i '/^>>>HIST_IRRIG<<</d'           $HISTRC
+else
+   sed -i 's/>>>HIST_IRRIG<<</''/g'        $HISTRC
+endif
+
+# for ensemble simulations, set 'GridComp' to ENSAVG 
+
 if($NENS > 1) then
    set GridComp = ENSAVG
    sed -i 's|VEGDYN|'VEGDYN_e0000'|g' $HISTRC
@@ -63,16 +81,6 @@ if($NENS > 1) then
 #   sed -i 's|DATAATM|'DATAATM0000'|g' $HISTRC
 endif
 
-sed -i 's|GridComp|'$GridComp'|g' $HISTRC
+# fill in source 'GridComp' information for output variables
 
-if($AEROSOL_DEPOSITION == 0) then
-   sed -i '/^>>>HIST_AEROSOL<<</d' $HISTRC
-else
-   sed -i 's/>>>HIST_AEROSOL<<</''/g' $HISTRC
-endif
-
-if($RUN_IRRIG == 0) then
-   sed -i '/^>>>HIST_IRRIG<<</d' $HISTRC
-else
-   sed -i 's/>>>HIST_IRRIG<<</''/g' $HISTRC
-endif
+sed -i 's|GridComp|'$GridComp'|g' $HISTRC

GEOSldas_App/util/postproc/ObsFcstAna_stats/Get_ObsFcstAna_stats.py → GEOSldas_App/util/postproc/ObsFcstAna_stats/Get_ObsFcstAna_sums.py

@@ -2,24 +2,21 @@
 
 """
 Sample script for post-processing GEOSldas ObsFcstAna output into data assimilation diagnostics.
-First, compute and store monthly sums and sums of squares and cross-products of raw ObsFcstAna output.  
+Computes and stores monthly sums and sums of squares and cross-products of raw ObsFcstAna output.  
 Data assimilation diagnostics ("stats") such as the mean and std-dev of the observation-minus-forecast 
-residuals can then be diagnosed quickly from these intermediate "sums" files. 
-Sample script optionally computes and plots:
-- Maps of long-term data assimilation diagnostics (see also Plot_stats_maps.py).
-- Monthly time series of spatially averaged data assimilation diagnostics (see also Plot_stats_timeseries.py).
+residuals can then be diagnosed quickly from these intermediate "sums" files.  See Plot_stats_*.py.
 
 Usage:
     1. Edit "user_config.py" with experiments information.
     2. Run this script as follows (on Discover):
     $ module load python/GEOSpyD
-    $ ./Get_ObsFcstAna_stats.py
+    $ ./Get_ObsFcstAna_sums.py
 
     # Background execution:
-    $ nohup ./Get_ObsFcstAna_stats.py > out.log &
+    $ nohup ./Get_ObsFcstAna_sums.py > out.log &
 
 Authors: Q. Liu, R. Reichle, A. Fox
-Last Modified: May 2025
+Last Modified: June 2025
 """
 
 import sys;       sys.path.append('../../shared/python/')
@@ -64,33 +61,6 @@ def main():
     # Compute and save monthly sums
     postproc.save_monthly_sums()
 
-    # --------------------------------------------------------------------------------------
-    # Optionally compute long-term temporal/spatial statistics and create plots.
-    # The plotting scripts can also run standalone using the individual Plot_stats_*.py scripts,
-    #   as long as the monthly sum files are available.
-
-    plot_maps       = False
-    plot_timeseries = False
-
-    if plot_maps:            # Compute long-term temporal stats and plot maps
-
-        stats_file  = out_path + 'temporal_stats_'+exp_list[0]['exptag']+ '_'+start_time.strftime('%Y%m%d')+'_'+ \
-            (end_time+timedelta(days=-1)).strftime('%Y%m%d')+'.nc4'
-
-        # temporal_stats is a dictionary that contains all mean/variances fields for computing long-term O-F/O-A stats
-        # each field has the dimension [N_tile, N_species]
-
-        temporal_stats = postproc.calc_temporal_stats_from_sums(write_to_nc=True, fout_stats=stats_file)
-
-        # Example to plot some O-F maps
-        from Plot_stats_maps import plot_OmF_maps
-        plot_OmF_maps(postproc, temporal_stats, fig_path=out_path )
-
-    if plot_timeseries:      # Compute spatially averaged stats and plot monthly time series of stats
-
-        from Plot_stats_timeseries import Plot_monthly_OmF_bars
-        Plot_monthly_OmF_bars(postproc, fig_path=out_path)
-
 if __name__ == '__main__':
     main()